2015-09-24

Using openssl to test two way SSL connectivity

In my previous post Building a two way https web service server using Java, I said I used openssl to do the web service server testing.

In fact, before choosing openssl, I have tried other methods like curl, various Chrome extensions.  But finally I stick to openssl because of its flexibility and availability of low level output to facilitate my debugging.

In fact the script is as simple as follows:

openssl s_client -connect localhost:8000 -cert client.pem -key key.pem -CApath . -CAfile ca.pem -showcerts -debug -msg -state -crlf -ign_eof <<EOF
POST /app HTTP/1.1
hostname: localhost
Content-Type: application/x-www-form-urlencoded
Content-Length: XXX

<?xml version='1.0'?><Envelope xmlns='http://schemas.xmlsoap.org/soap/envelope/' xmlns:op='http://schemas.xyz.com/svc' xmlns:ems='http://schemas.xyz.com/ems'><Header>Header_Text</Header><Body><op:AddPbs><ems:ChannelAcctId><ems:ChannelId>06</ems:ChannelId><ems:AcctId>1234567</ems:AcctId></ems:ChannelAcctId></op:AddPbs></Body></Envelope>

EOF

Let me explain each parameter one by one:

s_client: it is the SSL/TLS client program of openssl suite
connect: it designates the web service connection details (IP address and port)
cert: it specifies the self (client) certificate to present to the host
key: it specifies the private key file signing the client certificate
CApath: it designates the folder containing the certificate chain(s)
CAfile: it designates the "trusted" CA which signs the server certificate
showcerts: it shows the whole server certificate chain
debug: it is a treasure of openssl
msg: ditto
state: ditto
crlf: in Unix, the line delimiter is LF only.  This switch will cause the content to be transmitted with CR+LF instead (but please note the Content-Length has to be modified accordingly)
ign_eof: I originally omitted this switch.  But I find it is needed because otherwise the connection is closed too early by openssl before the server can send  the response back.


Building a two way https web service server using Java


I have the following requirements:
  • to build a multi-thread http server serving inbound web service requests via POST
  • the request is not conventional SOAP but is still XML-based
  • both client and server authentication (certificate based) is needed (riding not https)

 After some googling I found that Java provides a convenient framework to deliver this capability (as in Meisch's article "Java Webservice using HTTPS part 2")

The major classes used is:
  1. com.sun.net.httpserver.HttpsServer (abstract class); and
  2. com.sun.net.httpserver.HttpHandler (interface)
As HttpsServer is an abstract class, it cannot be instantiated (creating an object via new operator).  My example class java_https_server is therefore to implement the HttpHandler interface and have a method CreateHttpServer to create an HttpsServer object via the static HttpsServer.create() method.

The logic of the program is as follows:
  1. inside the main method, after creating a dummy java_https_server object, invoke the CreateHttpServer method to create a HttpsServer object
  2. inside CreateHttpServer method, open a Java Key Store containing the web server's own certificate and the Trust Store containing the trusted client certificate (the latter is needed because I use two-way SSL).  An SSLContext object instance is linked to these two key stores.  Finally, a context (which I find useless because I cannot still associate different handler with different context) is created for the HttpsServer object.
  3. the main method continue to set up a shutdown hook to graceful termination handling
  4. since the java_https_server implements the HttpHandler interface, a handle() method is defined  This is the core running logic of the whole web service server. I have added many println statements to display the attributes of the http connection in my codes.
The source listing is as follows:

import java.io.FileInputStream;

import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpsServer;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpsExchange;
import com.sun.net.httpserver.HttpsConfigurator;
import com.sun.net.httpserver.HttpsParameters;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.SSLContext;
import java.security.KeyStore;
import java.net.URI;
import java.net.InetSocketAddress;

public class java_https_server implements HttpHandler {
private static final int HTTP_OK_STATUS = 200;
// ----------class property --------------
private String context = "/";
private int port = 8000;
private String keystorePasswordString = "password";
private String keystoreFile = "/full_path/keystore.jks";
private String truststorePasswordString = "password";
private String truststoreFile = "/full_path/truststore.jks";
// --------- Constructor -------------------
public java_https_server () {
  }
// --------------------------------------------
public HttpsServer CreateHttpServer(int port, String context) {
  HttpsServer httpServer;
  try {
    httpServer = HttpsServer.create(new InetSocketAddress(port), 0);
    SSLContext sslContext = SSLContext.getInstance("TLS");
    // server keystore
    char[] keystorePassword = keystorePasswordString.toCharArray();
    KeyStore ks = KeyStore.getInstance("JKS");
    ks.load (new FileInputStream(keystoreFile), keystorePassword);
    KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
    kmf.init (ks, keystorePassword);
    // server truststore
    TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
    char[] truststorePassword = truststorePasswordString.toCharArray();
    ks.load (new FileInputStream(truststoreFile), truststorePassword);
    tmf.init (ks);
    sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
    // create an anonymous inner class HttpsConfigurator to require client certificate
    HttpsConfigurator configurator = new HttpsConfigurator(sslContext) {
      public void configure (HttpsParameters params) {
        SSLParameters sslParams = getSSLContext().getDefaultSSLParameters();
        sslParams.setNeedClientAuth(true);
        params.setSSLParameters(sslParams);
        }
      };
    httpServer.setHttpsConfigurator(configurator);
    //Create a new context for the given context and handler
    httpServer.createContext(context, this);
    //Create a default executor
    httpServer.setExecutor(null);
    }
  catch (Exception e) {
    e.printStackTrace();
    return null;
    }
  return httpServer;
  } // method CreateHttpServer
// --------------------------------------------
public void handle(HttpExchange t) throws IOException {
  URI uri = t.getRequestURI();
  String response;
  System.out.println ("LocalAddress: " + t.getLocalAddress().toString());
  System.out.println ("RemoteAddress: " + t.getRemoteAddress().toString());
  System.out.println ("URL is: " + uri.toString());
  System.out.println ("Method: " + t.getRequestMethod());
  System.out.println ("Client Certficate: " +
    ((HttpsExchange)t).getSSLSession().getPeerCertificateChain()[0].getSubjectDN());
  if (t.getRequestMethod().equals ("POST")) {
    InputStream is = t.getRequestBody();
    byte[] data = new byte[100000];
    int length = is.read(data);
    if (length == 100000)
      System.out.println ("Warning: the input buffer is FULL!");
    System.out.println ("Request Length: " + length);
    data = java.util.Arrays.copyOf(data, length); // trim the array to the correct size
    System.out.println ("Request Body:[" + new String(data) + "]");
    is.close();
    response = "Give your response here";
    }
  else {
    response = "Error";
    }
  //Set the response header status and length
  t.sendResponseHeaders(HTTP_OK_STATUS, response.getBytes().length);
  //Write the response string
  OutputStream os = t.getResponseBody();
  os.write(response.getBytes());
  os.close();
}
// --------------------------------------------
public static void main(String[] args) throws Exception {
  java_https_server server = new java_https_server();
  System.out.println("Use Ctrl-C (foreground) or \"kill -15 (background)\" to stop me");
  final HttpsServer httpServer = server.CreateHttpServer(server.port, server.context);
  Runtime.getRuntime().addShutdownHook(new Thread() {
    @Override
    public void run() {
      System.out.println("Stopping Server..");
      httpServer.stop(0);
      System.out.println("Server stopped");
      } // run()
    });
  httpServer.start(); // Start the server
  System.out.println("Server is started and listening on port "+ server.port);
  } // method main
} // class java_https_server

Some example runtime output is as follows:
$ java -cp . java_https_server
Use Ctrl-C (foreground) or "kill -15 (background)" to stop me
Server is started and listening on port 8000
LocalAddress: /127.0.0.1:8000
RemoteAddress: localhost/127.0.0.1:52204
URL is: /app
Method: POST
Client Certficate: CN=ccm010, OU=xxx, O=xxx, L=xxx, ST=xxx, C=HK
Request Length: 385
Request Body:[<?xml version='1.0'?><Envelope xmlns='http://schemas.xmlsoap.org/soap/envelope/' xmlns:op='http://schemas.xyz.com/svc' xmlns:ems='http://schemas.xyz.com/ems'><Header>Header_Text</Header><Body><op:AddPbs><ems:ChannelAcctId><ems:ChannelId>06</ems:ChannelId><ems:AcctId>1234567</ems:AcctId></ems:ChannelAcctId></op:AddPbs></Body></Envelope>]
Stopping Server..
Server stopped

I will cover how to test using openssl with client certificate in the next post.

Using openssl to test two way SSL connectivity


2015-06-30

Using libxml for XPath

With the proliferation of presentation of data using XML, I find parsing the data is not easy as more and more ingredients (like attributes, name spaces) are introduced.  Up to now, I still find using XPath syntax to represent XML nodes is the best because it simplifies the complicated data hierarchy structure into the conventional slash form, like:
/A/B/C
to select the C element in the XML
<A>
  <B>
    <C/>
  </B>
</A>
(example borrowed from https://en.wikipedia.org/wiki/XPath)

Although I have used Java's built-in XPath features previously, when I have a C project, I need to resort to external library for the job.  I finally choose libxml.  One of the headaches in libxml is its memory management because otherwise you will induce memory leakage easily.
This document describes my learning.

This source code of my example is xpath_demo.c and the full listing is included in another post (link).

Compilation
Most of the installation of libxml is at /usr/local, therefore the sample program xpath_demo.c is compiled with the following switches:
cc -o xpath_demo -L/usr/local/lib -R/usr/local/lib -lxml2 -I/usr/local/include/libxml2 xpath_demo.c

Program Structure
The program has only two functions, the main function (which includes most of the logic) and register_namespaces (which is copied from libxml site for the name space registration)

Program Usage
The simplest usage is:
xpath_demo xml_filename xpath_expression
If there is name space, then the usage will be:
xpath_demo xml_filename xpath_expression name_space_list

Pseudo Codes

Invoke libxml function
Input/Output
Outstanding Object
xmlParseFile
Input: xml filename
Output: xmlDocPtr
xmlDocPtr
xmlPathNewContext
Input: xmlDocPtr
Output: xmlXPathContextPtr
xmlDocPtr
xmlXPathContextPtr
xmlXPathRegisterNs (only applicable for xml with namespace)
Input: xmlXPathContextPtr
namespace_prefix
namespace_URL

xmlXPathEvalExpression
Input: XPath_Expression,
xmlXPathContextPtr
Output: xmlXPathObjectPtr
xmlDocPtr
xmlXPathContextPtr
xmlXPathObjectPtr
xmlXPathFreeContext

xmlDocPtr
xmlXPathObjectPtr
Check if xmlXPathNodeSetIsEmpty
Input: xmlXPathObjectPtr->nodesetval

Retrieve the node:
xmlXPathObjectPtr ->nodesetval->nodeTab[0]


Retrieve the text of the node
xmlNodeGetContent
Input: xmlNode *
Output: xmlChar *
xmlDocPtr
xmlXPathObjectPtr
xmlChar * node_text
xmlFree (node_text)
xmlXPathFreeObject(xmlXPathObjectPtr)

xmlDocPtr
Final Clean up
xmlFreeDoc(xmlDocPtr);
xmlCleanupParser();

Nil

The "simplified" print out of various inputs are shown as follows:
cat data.xml
<?xml version='1.0'?>
<Envelope>
<Header>Header_Text</Header>
<Body attribute1='funny'>
<Field1>Value1</Field1>
<Field2>Value2</Field2>
</Body>
</Envelope>

xpath_demo error.xml /FIELD1
I/O warning : failed to load external entity "error.xml"
Error: Document not parsed successfully.

xpath_demo data.xml /Envelope/Body
node-text: "
Value1
Value2
"
Remark: According to the specification, the text of node includes all the text of its daughter nodes as well.

xpath_demo data.xml /Envelope/Body/Field1
node-text: "Value1"

xpath_demo data.xml /Envelope/Body/Field3
Empty

Cases with Name Space
Personally I do not like name space in XML because it is awkward.  Anway, libxml does support it, with an additional step to register the name space list.

An XML file (ns_data.xml) with name space is shown below:

<?xml version='1.0'?>
<Envelope xmlns:ns1='http://www.domain.com/ns/sample'>
<Header>Header_Text</Header>
<Body name='value'>
<ns1:Field1>Value1 in ns1</ns1:Field1>
<Field1>Value1 without NS<Field1>
</Body>
</Envelope>

A nameapace ns1 is defined in the root element <Envelope>.  You can see there are two tags with name Field1, one of which with name space ns1.  They are can accessed separately as follows:

xpath_demo ns_data.xml /Envelope/Body/ns1:Field1 ns1=http://www.domain.com/ns/sample
node-text: "Value1 in ns1"

xpath_demo ns_data.xml /Envelope/Body/Field1 ns1=http://www.domain.com/ns/sample

node-text: "Value1 without NS"

xpath_demo.c

/*
File : xpath_demo.c
Description: A demo C program to print the text of a node from a xpath expression
Usage: xpath_demo.c xml_file xpath_expression
Dependence: libxml2
How to make:
cc -o xpath_demo -L/usr/local/lib -R/usr/local/lib -lxml2 -I/usr/local/include/libxml2 xpath_demo.c
*/

#include <libxml/parser.h>
#include <libxml/xpath.h>
#include <libxml/xpathInternals.h>   /* for function xmlXPathRegisterNs */
#include <assert.h>

/* Function Prototype */
int  register_namespaces(xmlXPathContextPtr xpathCtx, const xmlChar* nsList);

/* ===================================================================== */
int main(int argc, char **argv) {

xmlDocPtr doc;
/* xmlNodeSetPtr nodeset; */
xmlXPathObjectPtr result;
xmlNode *node;
xmlChar *node_text;
xmlXPathContextPtr context;
char *filename;
xmlChar *xpath_expression;
xmlChar *nsList;

if ((argc != 3) && (argc != 4)) {
  fprintf(stderr, "Usage: %s xml_file xpath_expression  [<known-ns-list>]\n", argv[0]);
  fprintf(stderr, "where <known-ns-list> is a list of known namespaces\n");
  fprintf(stderr, "in \"<prefix1>=<href1> <prefix2>=href2> ...\" format\n");
  return(1);
  }
filename = argv[1];
xpath_expression = (xmlChar*) argv[2];

fprintf (stderr, "DEBUG: LIBXML_VERSION is " LIBXML_VERSION_STRING "\n");

doc = xmlParseFile(filename);
if (doc == NULL ) {
  fprintf(stderr, "Error: Document not parsed successfully.\n");
  xmlCleanupParser();
  return 1;
  }

context = xmlXPathNewContext(doc);
if (context == NULL) {
  fprintf(stderr, "Error in xmlXPathNewContext\n");
  xmlFreeDoc(doc);
  xmlCleanupParser();
  return 2;
  }

if (argc == 4) {
  nsList = (xmlChar*) argv[3];
  if (register_namespaces(context, nsList) < 0) {
    fprintf(stderr,"Error: failed to register namespaces list \"%s\"\n", nsList);
    xmlXPathFreeContext(context);
    xmlFreeDoc(doc);
    xmlCleanupParser();
    return 3;
    }
  }
result = xmlXPathEvalExpression(xpath_expression, context);
xmlXPathFreeContext (context);
if (result == NULL) {
  fprintf(stderr, "Error in xmlXPathEvalExpression\n");
  xmlFreeDoc(doc);
  xmlCleanupParser();
  return 4;
  }

/*
xmlXPathEvalExpression() call returns a set of ALL the nodes that match the expression
We are only interested in the first node returned
*/

if (xmlXPathNodeSetIsEmpty(result->nodesetval)) {
  printf ("Empty\n");
  xmlXPathFreeObject(result);
  xmlFreeDoc(doc);
  xmlCleanupParser();
  return 5;
  }

/* Retrieve the data */
node = result->nodesetval->nodeTab[0];
node_text = xmlNodeGetContent(node);
/* xmlNodeGetContent retrieves the text values of all children too.  This is correct */

fprintf (stderr, "DEBUG: node-type: %d node-name: %s\n" ,node->type, node->name);
xmlAttr *attr = node->properties;
while ( attr ) {
  fprintf (stderr, "DEBUG: attribute-name:%s attribute-value:%s\n" , attr->name, attr->children->content);
  attr = attr->next;
  } /* while */
printf ("node-text: \"%s\"\n", node_text);
xmlFree (node_text);
xmlXPathFreeObject(result);

/* Final clean up */
xmlFreeDoc(doc);
xmlCleanupParser();
return (0);
} /* main */

/**************************************************************************************/
/* The following fucnction is extracted from http://www.xmlsoft.org/examples/xpath1.c */
/**************************************************************************************/

/**
 * register_namespaces:
 * @xpathCtx:           the pointer to an XPath context.
 * @nsList:             the list of known namespaces in
 *                      "<prefix1>=<href1> <prefix2>=href2> ..." format.
 *
 * Registers namespaces from @nsList in @xpathCtx.
 *
 * Returns 0 on success and a negative value otherwise.
 */
int
register_namespaces(xmlXPathContextPtr xpathCtx, const xmlChar* nsList) {
    xmlChar* nsListDup;
    xmlChar* prefix;
    xmlChar* href;
    xmlChar* next;

    assert(xpathCtx);
    assert(nsList);

    nsListDup = xmlStrdup(nsList);
    if(nsListDup == NULL) {
        fprintf(stderr, "Error: unable to strdup namespaces list\n");
        return(-1);
    }

    next = nsListDup;
    while(next != NULL) {
        /* skip spaces */
        while((*next) == ' ') next++;
        if((*next) == '\0') break;

        /* find prefix */
        prefix = next;
        next = (xmlChar*)xmlStrchr(next, '=');
        if(next == NULL) {
            fprintf(stderr,"Error: invalid namespaces list format\n");
            xmlFree(nsListDup);
            return(-1);
        }
        *(next++) = '\0';

        /* find href */
        href = next;
        next = (xmlChar*)xmlStrchr(next, ' ');
        if(next != NULL) {
            *(next++) = '\0';
        }
        /* do register namespace */
        if(xmlXPathRegisterNs(xpathCtx, prefix, href) != 0) {
            fprintf(stderr,"Error: unable to register NS with prefix=\"%s\" and href=\"%s\"\n", prefix, href);
            xmlFree(nsListDup);
            return(-1);
        }
    }

    xmlFree(nsListDup);
    return(0);
}