Make i2d_X509_AUX work if *pp = NULL.

When *pp is NULL, don't write garbage, return an unexpected pointer
or leak memory on error.

(Imported from upstream's 36c37944909496a123e2656ad1f651769a7cc72f.)

This calling convention...

Change-Id: Ic733092cfb942a3e1d3ceda6797222901ad55bef
Reviewed-on: https://boringssl-review.googlesource.com/7944
Reviewed-by: Adam Langley <agl@google.com>

crypto/x509/x_x509.c

@@ -55,6 +55,7 @@
  * copied and put under another distribution licence
  * [including the GNU Public Licence.] */
 
+#include <assert.h>
 #include <stdio.h>
 
 #include <openssl/asn1t.h>
@@ -204,12 +205,26 @@ X509 *d2i_X509_AUX(X509 **a, const unsigned char **pp, long length)
     return NULL;
 }
 
-int i2d_X509_AUX(X509 *a, unsigned char **pp)
+/*
+ * Serialize trusted certificate to *pp or just return the required buffer
+ * length if pp == NULL.  We ultimately want to avoid modifying *pp in the
+ * error path, but that depends on similar hygiene in lower-level functions.
+ * Here we avoid compounding the problem.
+ */
+static int i2d_x509_aux_internal(X509 *a, unsigned char **pp)
 {
     int length, tmplen;
     unsigned char *start = pp != NULL ? *pp : NULL;
+
+    assert(pp == NULL || *pp != NULL);
+
+    /*
+     * This might perturb *pp on error, but fixing that belongs in i2d_X509()
+     * not here.  It should be that if a == NULL length is zero, but we check
+     * both just in case.
+     */
     length = i2d_X509(a, pp);
-    if (length < 0 || a == NULL) {
+    if (length <= 0 || a == NULL) {
         return length;
     }
 
@@ -224,6 +239,42 @@ int i2d_X509_AUX(X509 *a, unsigned char **pp)
     return length;
 }
 
+/*
+ * Serialize trusted certificate to *pp, or just return the required buffer
+ * length if pp == NULL.
+ *
+ * When pp is not NULL, but *pp == NULL, we allocate the buffer, but since
+ * we're writing two ASN.1 objects back to back, we can't have i2d_X509() do
+ * the allocation, nor can we allow i2d_X509_CERT_AUX() to increment the
+ * allocated buffer.
+ */
+int i2d_X509_AUX(X509 *a, unsigned char **pp)
+{
+    int length;
+    unsigned char *tmp;
+
+    /* Buffer provided by caller */
+    if (pp == NULL || *pp != NULL)
+        return i2d_x509_aux_internal(a, pp);
+
+    /* Obtain the combined length */
+    if ((length = i2d_x509_aux_internal(a, NULL)) <= 0)
+        return length;
+
+    /* Allocate requisite combined storage */
+    *pp = tmp = OPENSSL_malloc(length);
+    if (tmp == NULL)
+        return -1; /* Push error onto error stack? */
+
+    /* Encode, but keep *pp at the originally malloced pointer */
+    length = i2d_x509_aux_internal(a, &tmp);
+    if (length <= 0) {
+        OPENSSL_free(*pp);
+        *pp = NULL;
+    }
+    return length;
+}
+
 void X509_get0_signature(ASN1_BIT_STRING **psig, X509_ALGOR **palg,
                          const X509 *x)
 {