rewrite the DNS API to permit setting multiple records of the same type on the same domain

e.g. multiple TXT records

fixes #333

CHANGELOG.md

@@ -15,10 +15,15 @@ ownCloud:
 
 * Downloading files you uploaded to ownCloud broke because of a change in ownCloud 8.
 
+DNS:
+
+* Internationalized Domain Names (IDNs) should now work in email. If you had custom DNS or custom web settings for internationalized domains, check that they are still working.
+* It is now possible to set multiple TXT and other types of records on the same domain in the control panel.
+* The custom DNS API was completely rewritten to support setting multiple records of the same type on a domain. Any existing client code using the DNS API will have to be rewritten. (Existing code will just get 404s back.)
+
 System:
 
 * Backups now use duplicity's built-in gpg symmetric AES256 encryption rather than my home-brewed encryption. Old backups will be incorporated inside the first backup after this update but then deleted from disk (i.e. your backups from the previous few days will be backed up).
-* Internationalized Domain Names (IDNs) should now work in email. If you had custom DNS or custom web settings for internationalized domains, check that they are still working.
 * All Mail-in-a-Box release tags are now signed on github, instructions for verifying the signature are added to the README, and the integrity of all non-Ubuntu packages downloaded during setup is now verified against a SHA1 hash stored in the tag itself.
 
 v0.08 (April 1, 2015)

management/daemon.py

@@ -233,36 +233,70 @@ def dns_set_secondary_nameserver():
 	except ValueError as e:
 		return (str(e), 400)
 
-@app.route('/dns/set')
+@app.route('/dns/custom')
 @authorized_personnel_only
-def dns_get_records():
+def dns_get_records(qname=None, rtype=None):
 	from dns_update import get_custom_dns_config
-	return json_response([{
+	return json_response([
+	{
 		"qname": r[0],
 		"rtype": r[1],
 		"value": r[2],
-		} for r in get_custom_dns_config(env) if r[0] != "_secondary_nameserver"])
+	}
+	for r in get_custom_dns_config(env)
+	if r[0] != "_secondary_nameserver"
+		and (not qname or r[0] == qname)
+		and (not rtype or r[1] == rtype) ])
 
-@app.route('/dns/set/<qname>', methods=['POST'])
+@app.route('/dns/custom/<qname>', methods=['GET', 'POST', 'PUT', 'DELETE'])
-@app.route('/dns/set/<qname>/<rtype>', methods=['POST'])
+@app.route('/dns/custom/<qname>/<rtype>', methods=['GET', 'POST', 'PUT', 'DELETE'])
-@app.route('/dns/set/<qname>/<rtype>/<value>', methods=['POST'])
 @authorized_personnel_only
-def dns_set_record(qname, rtype="A", value=None):
+def dns_set_record(qname, rtype="A"):
 	from dns_update import do_dns_update, set_custom_dns_record
 	try:
-		# Get the value from the URL, then the POST parameters, or if it is not set then
+		# Normalize.
-		# use the remote IP address of the request --- makes dynamic DNS easy. To clear a
+		rtype = rtype.upper()
-		# value, '' must be explicitly passed.
+
-		if value is None:
+		# Read the record value from the request BODY, which must be
-			value = request.form.get("value")
+		# ASCII-only. Not used with GET.
-		if value is None:
+		value = request.stream.read().decode("ascii", "ignore").strip()
+
+		if request.method == "GET":
+			# Get the existing records matching the qname and rtype.
+			return dns_get_records(qname, rtype)
+
+		elif request.method in ("POST", "PUT"):
+			# There is a default value for A/AAAA records.
+			if rtype in ("A", "AAAA") and value == "":
 				value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
-		if value == '' or value == '__delete__':
+
-			# request deletion
+			# Cannot add empty records.
+			if value == '':
+				return ("No value for the record provided.", 400)
+
+			if request.method == "POST":
+				# Add a new record (in addition to any existing records
+				# for this qname-rtype pair).
+				action = "add"
+			elif request.method == "PUT":
+				# In REST, PUT is supposed to be idempotent, so we'll
+				# make this action set (replace all records for this
+				# qname-rtype pair) rather than add (add a new record).
+				action = "set"
+		
+		elif request.method == "DELETE":
+			if value == '':
+				# Delete all records for this qname-type pair.
 				value = None
-		if set_custom_dns_record(qname, rtype, value, "set", env):
+			else:
-			return do_dns_update(env) or "No Change"
+				# Delete just the qname-rtype-value record exactly.
+				pass
+			action = "remove"
+
+		if set_custom_dns_record(qname, rtype, value, action, env):
+			return do_dns_update(env) or "Something isn't right."
 		return "OK"
+
 	except ValueError as e:
 		return (str(e), 400)
 

management/templates/custom-dns.html

@@ -93,44 +93,56 @@
 
 <p>Use your box&rsquo;s DNS API to set custom DNS records on domains hosted here. For instance, you can create your own dynamic DNS service.</p>
 
-<p>Send a POST request like this:</p>
+<p>Usage:</p>
 
-<pre>curl -d "" --user {email}:{password} https://{{hostname}}/admin/dns/set/<b>qname</b>[/<b>rtype</b>[/<b>value</b>]]</pre>
+<pre>curl -X <b>VERB</b> [-d "<b>value</b>"] --user {email}:{password} https://{{hostname}}/admin/dns/custom[/<b>qname</b>[/<b>rtype</b>]]</pre>
 
-<h4>HTTP POST parameters</h4>
+<p>(Brackets denote an optional argument.)</p>
+
+<h4>Verbs</h4>
+
+<table class="table">
+<thead><th>Verb</th> <th>Usage</th></thead>
+<tr><td>GET</td> <td>Returns matching custom DNS records as a JSON array of objects. Each object has the keys <code>qname</code>, <code>rtype</code>, and <code>value</code>. The optional <code>qname</code> and <code>rtype</code> parameters in the request URL filter the records returned in the response. The request body (<code>-d "..."</code>) must be omitted.</td></tr>
+<tr><td>PUT</td> <td>Sets a custom DNS record replacing any existing records with the same <code>qname</code> and <code>rtype</code>. Use PUT (instead of POST) when you only have one value for a <code>qname</code> and <code>rtype</code>, such as typical <code>A</code> records (without round-robin).</td></tr>
+<tr><td>POST</td> <td>Adds a new custom DNS record. Use POST when you have multiple <code>TXT</code> records or round-robin <code>A</code> records. (PUT would delete previously added records.)</td></tr>
+<tr><td>DELETE</td> <td>Deletes custom DNS records. If the request body (<code>-d "..."</code>) is empty or omitted, deletes all records matching the <code>qname</code> and <code>rtype</code>. If the request body is present, deletes only the record matching the <code>qname</code>, <code>rtype</code> and value.</td></tr>
+</table>
+
+<h4>Parameters</h4>
 
 <table class="table">
 <thead><th>Parameter</th> <th>Value</th></thead>
 <tr><td>email</td> <td>The email address of any administrative user here.</td></tr>
 <tr><td>password</td> <td>That user&rsquo;s password.</td></tr>
-<tr><td>qname</td> <td>The fully qualified domain name for the record you are trying to set.</td></tr>
+<tr><td>qname</td> <td>The fully qualified domain name for the record you are trying to set. It must be one of the domain names or a subdomain of one of the domain names hosted on this box. (Add mail users or aliases to add new domains.)</td></tr>
-<tr><td>rtype</td> <td>The resource type. <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), or <code>CNAME</code> (an alias, which is a fully qualified domain name).</td></tr>
+<tr><td>rtype</td> <td>The resource type. Defaults to <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), <code>CNAME</code> (an alias, which is a fully qualified domain name &mdash; don&rsquo;t forget the final period), <code>MX</code>, or <code>SRV</code>.</td></tr>
-<tr><td>value</td> <td>The new record&rsquo;s value. If omitted, the IPv4 address of the remote host is used. This is handy for dynamic DNS! To delete a record, use &ldquo;__delete__&rdquo;.</td></tr>
+<tr><td>value</td> <td>For PUT, POST, and DELETE, the record&rsquo;s value. If the <code>rtype</code> is <code>A</code> or <code>AAAA</code> and <code>value</code> is empty or omitted, the IPv4 or IPv6 address of the remote host is used (be sure to use the <code>-4</code> or <code>-6</code> options to curl). This is handy for dynamic DNS!</td></tr>
 </table>
 
-<p style="margin-top: 1em">Note that <code>-d ""</code> is merely to ensure curl sends a POST request. You do not need to put anything inside the quotes. You can also pass the value using typical form encoding in the POST body.</p>
-
 <p>Strict <a href="http://tools.ietf.org/html/rfc4408">SPF</a> and <a href="https://datatracker.ietf.org/doc/draft-kucherawy-dmarc-base/?include_text=1">DMARC</a> records will be added to all custom domains unless you override them.</p>
 
 <h4>Examples:</h4>
 
+<p>Try these examples. For simplicity the examples omit the <code>--user me@mydomain.com:yourpassword</code> command line argument which you must fill in with your email address and password.</p>
+
 <pre># sets laptop.mydomain.com to point to the IP address of the machine you are executing curl on
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/laptop.mydomain.com
+curl -X PUT https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# sets an alias
+# deletes that record and all A records for that domain name
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/cname/bar.mydomain.com
+curl -X DELETE https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# clears the alias
+# sets a CNAME alias
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/bar.mydomain.com/cname/__delete__
+curl -X PUT -d "bar.mydomain.com." https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a TXT record using the alternate value syntax
+# deletes that CNAME and all CNAME records for that domain name
-curl -d "value=something%20here" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/txt
+curl -X DELETE https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a <a href="http://en.wikipedia.org/wiki/SRV_record">SRV record</a> for the "service" and "protocol" hosted on "target" server
+# adds a TXT record using POST to preserve any previous TXT records
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.{{hostname}}/srv/"priority weight port target"
+curl -X POST -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
 
-# sets a SRV record using the value syntax
+# deletes that one TXT record while preserving other TXT records
-curl -d "value=priority weight port target" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.host/srv
+curl -X DELETE -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
 </pre>
 
 <script>
@@ -161,7 +173,7 @@ function show_custom_dns() {
 
 function show_current_custom_dns() {
   api(
-    "/dns/set",
+    "/dns/custom",
     "GET",
     { },
     function(data) {
@@ -176,6 +188,7 @@ function show_current_custom_dns() {
         $('#custom-dns-current').find("tbody").append(tr);
         tr.attr('data-qname', data[i].qname);
         tr.attr('data-rtype', data[i].rtype);
+        tr.attr('data-value', data[i].value);
         tr.append($('<td class="long"/>').text(data[i].qname));
         tr.append($('<td/>').text(data[i].rtype));
         tr.append($('<td class="long"/>').text(data[i].value));
@@ -187,7 +200,8 @@ function show_current_custom_dns() {
 function delete_custom_dns_record(elem) {
   var qname = $(elem).parents('tr').attr('data-qname');
   var rtype = $(elem).parents('tr').attr('data-rtype');
-  do_set_custom_dns(qname, rtype, "__delete__");
+  var value = $(elem).parents('tr').attr('data-value');
+  do_set_custom_dns(qname, rtype, value, "DELETE");
   return false;
 }
 
@@ -208,7 +222,7 @@ function do_set_secondary_dns() {
     });
 }
 
-function do_set_custom_dns(qname, rtype, value) {
+function do_set_custom_dns(qname, rtype, value, method) {
   if (!qname) {
     if ($('#customdnsQname').val() != '')
       qname = $('#customdnsQname').val() + '.' + $('#customdnsZone').val();
@@ -216,14 +230,13 @@ function do_set_custom_dns(qname, rtype, value) {
       qname = $('#customdnsZone').val();
     rtype = $('#customdnsType').val();
     value = $('#customdnsValue').val();
+    method = 'POST';
   }
 
   api(
-    "/dns/set/" + qname + "/" + rtype,
+    "/dns/custom/" + qname + "/" + rtype,
-    "POST",
+    method,
-    {
+    value,
-      value: value
-    },
     function(data) {
       if (data == "") return; // nothing updated
       show_modal_error("Custom DNS", $("<pre/>").text(data));

management/templates/index.html

@@ -337,6 +337,11 @@ function api(url, method, data, callback, callback_error) {
     method: method,
     cache: false,
     data: data,
+
+    // the custom DNS api sends raw POST/PUT bodies --- prevent URL-encoding
+    processData: typeof data != "string",
+    mimeType: typeof data == "string" ? "text/plain; charset=ascii" : null,
+
     beforeSend: function(xhr) {
       // We don't store user credentials in a cookie to avoid the hassle of CSRF
       // attacks. The Authorization header only gets set in our AJAX calls triggered