implrep.js

/* Process assertions in index.html, combine with configuration
 * and test results data, and generate an implementation report.   
 * See testing/README.md
 */

// Dependencies
const path = require('path');
const fs = require('fs');
const cheerio = require('cheerio');
const csvtojson=require('csvtojson'); // V2

// Parameters
//=======================================================================
const config = JSON.parse(fs.readFileSync("./implrep_config.json").toString());

//==== Flags

// include test specs appendix in report
const show_test_specs = (undefined != config.show_test_specs) ?
  config.show_test_specs : false;                                       

// include interop test results in report
const show_interop_results = (undefined != config.show_interop_results) ?
  config.show_interop_results : false;                                  

//==== Directories

// source directory (location of specification)
const src_dir = (undefined != config.src_dir) ?
  path.join(__dirname,config.src_dir) : __dirname;

// testing directory 
const testing_dir_rel = (undefined != config.testing_dir_rel) ?
  config.testing_dir_rel : "testing";                            
const testing_dir = (undefined != config.testing_dir) ?
  path.join(__dirname,config.testing_dir) : path.join(src_dir,testing_dir_rel);                            

// target directory for report output
const report_dir = (undefined != config.report_dir) ?
  path.join(__dirname,config.report_dir) : testing_dir;                                      

// location of other inputs
const inputs_dir_rel = (undefined != config.inputs_dir_rel) ?
  config.inputs_dir_rel : "inputs";                            
const inputs_dir = (undefined != config.inputs_dir) ?
  path.join(__dirname,config.inputs_dir) : path.join(testing_dir, inputs_dir_rel);                 

// location of HTML templates
const templates_dir_rel = (undefined != config.templates_dir_rel) ?
  config.templates_dir_rel : "templates";                            
const templates_dir = (undefined != config.templates_dir) ?
  path.join(__dirname,config.templates_dir) : path.join(inputs_dir, templates_dir_rel);            

// implementation descriptions
const descs_dir_rel = (undefined != config.descs_dir_rel) ?
  config.descs_dir_rel : "implementations";                            
const descs_dir = (undefined != config.descs_dir) ?
  path.join(__dirname,config.descs_dir) : path.join(inputs_dir, descs_dir_rel);          

// test results for each assertion and impl
const results_dir_rel = (undefined != config.results_dir_rel) ?
  config.results_dir_rel : "results";                            
const results_dir = (undefined != config.results_dir) ?
  path.join(__dirname,config.results_dir) : path.join(inputs_dir, results_dir_rel);                

// interop test results directory
const interop_dir_rel = (undefined != config.interop_dir_rel) ?
  config.interop_dir_rel : "interop";                            
const interop_dir = (undefined != config.interop_dir) ?
  path.join(__dirname,config.interop_dir) : path.join(inputs_dir, interop_dir_rel);                

//==== Inputs

// source specification (rendered)
const src_htmlfile_rel = (undefined != config.src_htmlfile_rel) ?
  config.src_htmlfile_rel : "index.html";               
const src_htmlfile = (undefined != config.src_htmlfile) ?
  path.join(__dirname,config.src_htmlfile) : path.join(src_dir, src_htmlfile_rel);               

// report template
const rt_htmlfile_rel = (undefined != config.rt_htmlfile_rel) ?
  config.rt_htmlfile_rel : "report.html";               
const rt_htmlfile = (undefined != config.rt_htmlfile) ?
  path.join(__dirname,config.rt_htmlfile) : path.join(templates_dir, rt_htmlfile_rel);         

// testspec template
const tt_htmlfile_rel = (undefined != config.tt_htmlfile_rel) ?
  config.tt_htmlfile_rel : "testspec.html";       
const tt_htmlfile = (undefined != config.tt_htmlfile) ?
  path.join(__dirname,config.tt_htmlfile) : path.join(templates_dir, tt_htmlfile_rel);       

// interop template
const it_htmlfile_rel = (undefined != config.it_htmlfile_rel) ?
  config.it_htmlfile_rel : "interop.html";        
const it_htmlfile = (undefined != config.it_htmlfile) ?
  path.join(__dirname,config.it_htmlfile) : path.join(templates_dir, it_htmlfile_rel);        

// test specifications for assertions
const ts_htmlfile_rel = (undefined != config.ts_htmlfile_rel) ?
  config.ts_htmlfile_rel : "testspec.html";          
const ts_htmlfile = (undefined != config.ts_htmlfile) ?
  path.join(__dirname,config.ts_htmlfile) : path.join(inputs_dir, ts_htmlfile_rel);          

// extra non-spec assertions (e.g. child assertions)
const ea_htmlfile_rel = (undefined != config.ea_htmlfile_rel) ?
  config.ea_htmlfile_rel : "extra-asserts.html";     
const ea_htmlfile = (undefined != config.ea_htmlfile) ?
  path.join(__dirname,config.ea_htmlfile) : path.join(inputs_dir, ea_htmlfile_rel);     

// assertion dependencies
const depends_csvfile_rel = (undefined != config.depends_csvfile_rel) ?
  config.depends_csvfile_rel : "depends.csv";        
const depends_csvfile = (undefined != config.depends_csvfile) ?
  path.join(__dirname,config.depends_csvfile) : path.join(inputs_dir, depends_csvfile_rel);        

// assertion categories
const categories_csvfile_rel = (undefined != config.categories_csvfile_rel) ?
  config.categories_csvfile_rel : "categories.csv";  
const categories_csvfile = (undefined != config.categories_csvfile) ?
  path.join(__dirname,config.categories_csvfile) : path.join(inputs_dir, categories_csvfile_rel);  

// at-risk assertions (input)
const atrisk_csvfile_rel = (undefined != config.atrisk_csvfile_rel) ?
  config.atrisk_csvfile_rel : "atrisk.csv";          
const atrisk_csvfile = (undefined != config.atrisk_csvfile) ?
  path.join(__dirname,config.atrisk_csvfile) : path.join(inputs_dir, atrisk_csvfile_rel);          

// structured implementation data
const impls_csvfile_rel = (undefined != config.impls_csvfile_rel) ?
  config.impls_csvfile_rel : "impl.csv";             
const impls_csvfile = (undefined != config.impls_csvfile) ?
  path.join(__dirname,config.impls_csvfile) : path.join(inputs_dir, impls_csvfile_rel);             

// assertions to be tested manually
const manual_csvfile_rel = (undefined != config.manual_csvfile_rel) ?
  config.manual_csvfile_rel : "manual.csv";         
const manual_csvfile = (undefined != config.manual_csvfile) ?
  path.join(__dirname,config.manual_csvfile) : path.join(testing_dir, manual_csvfile_rel);         

// assertions to be suppressed from IR
const suppressed_csvfile_rel = (undefined != config.suppressed_csvfile_rel) ?
  config.suppressed_csvfile_rel : "suppressed.csv"; 
const suppressed_csvfile = (undefined != config.suppressed_csvfile) ?
  path.join(__dirname,config.suppressed_csvfile) : path.join(testing_dir, suppressed_csvfile_rel); 

//==== Outputs 

// report filename
const report_htmlfile_rel = (undefined != config.report_htmlfile_rel) ?
  config.report_htmlfile_rel : "report.html";
const report_htmlfile = (undefined != config.report_htmlfile) ?
  path.join(__dirname,config.report_htmlfile) : path.join(report_dir, report_htmlfile_rel);

// all assertions
const report_csvfile_rel = (undefined != config.report_csvfile_rel) ?
  config.report_csvfile_rel : "template.csv";
const report_csvfile = (undefined != config.report_csvfile) ?
  path.join(__dirname,config.report_csvfile) : path.join(report_dir, report_csvfile_rel);

// at-risk assertion styling
const atrisk_cssfile_rel = (undefined != config.atrisk_cssfile_rel) ?
  config.atrisk_cssfile_rel : "atrisk.css";         
const atrisk_cssfile = (undefined != config.atrisk_cssfile) ?
  path.join(__dirname,config.atrisk_cssfile) : path.join(testing_dir, atrisk_cssfile_rel);         

// at-risk assertions (output)
const atrisk_out_csvfile_rel = (undefined != config.atrisk_out_csvfile_rel) ?
  config.atrisk_out_csvfile_rel : "atrisk.csv";         
const atrisk_out_csvfile = (undefined != config.atrisk_out_csvfile) ?
  path.join(__dirname,config.atrisk_out_csvfile) : path.join(testing_dir, atrisk_out_csvfile_rel);     

// Base URL to use for internal links (necessary since we use <base> in
// the HTML template to resolve hyperlinks included from index.html,
// so this needs to link back to the report from the location of that
// file...
const report_base = (undefined != config.report_base) ?
  config.report_base : ""; 
// path.join(report_dir, "report.html");

// Base URL for specification. 
// const src_base = "https://www.w3.org/TR/wot-thing-description";
const src_base = (undefined != config.src_base) ?
  config.src_base : "https://w3c.github.io/wot-thing-description";

// Whether or not to duplicate category and assertion in test spec appendix.
// Off by default since it is redundant, but is convenient sometimes.
const repeat_assertion_in_appendix = (undefined != config.repeat_assertion_in_appendix) ?
  config.repeat_assertion_in_appendix : false;

//==== Other

// Verbosity level
const verbosity = (undefined != config.verbosity) ?
  config.verbosity : 0;

//=======================================================================

// Map to track dependencies
var depends = new Map();

// Verbosity flags
const silent_v = (verbosity == 0);
const warn_v = (verbosity >= 1);
const info_v = (verbosity >= 2);
const chatty_v = (verbosity >= 3);
const debug_v = (verbosity >= 4);
//=======================================================================

// Read in test specs and store as a map
// (Synchronous)
const ts_raw = fs.readFileSync(ts_htmlfile,'UTF-8');
var ts_dom = cheerio.load(ts_raw);
var testspec = {};
ts_dom('span[class="testspec"]').each(function(i,elem) {
    let id = ts_dom(this).attr('id');
    if (undefined === id) {
        if (warn_v) console.log("Warning: testspec without id:\n",ts_dom(this).html());
    } else {
        if (chatty_v) console.log("Adding testspec for",id);
        if (debug_v) console.log("  testspec is:\n",ts_dom(this).html());
        testspec[id] = ts_dom(this);
    }
});
if (debug_v) console.log("test specs:",ts_dom.html());

// Read in implementation descriptions, store as a map from name to doms
// (Synchronous)
var descs = {};
var desc_names = {};
var desc_files = fs.readdirSync(descs_dir);
function get_descs() {
   for (let fi=0; fi<desc_files.length; fi++) {
       var file = path.join(descs_dir, desc_files[fi]);
       if (file.match(/.html$/g)) {
           if (info_v) console.log("processing implementation descriptions in",file);
           let desc_raw = fs.readFileSync(file,'UTF-8');
           let desc_dom = cheerio.load(desc_raw);
           desc_dom('div[class="impl"]').each(function(i,elem) {
               let id = desc_dom(this).attr('id');
               if (undefined === id) {
                   console.log("Warning: implementation description file without id:\n",desc_dom(this).html());
               } else {
                   if (chatty_v) console.log("Adding implementation description for",id);
                   if (debug_v) console.log("  implementation description is:\n",desc_dom(this).html());
                   descs[id] = desc_dom(this);
                   desc_names[id] = desc_dom("h4").text();
               }
           });
       }
   } 
}
get_descs();

// Initialize report document dom with template
// (Synchronous)
var report_template_raw = fs.readFileSync(rt_htmlfile, 'UTF-8');
const testspec_template_raw = fs.readFileSync(tt_htmlfile, 'UTF-8');
const interop_template_raw = fs.readFileSync(it_htmlfile, 'UTF-8');

// Set version date

const monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
let lastModified = new Date();
let lastModifiedString = lastModified.getDate() + ' ' + monthNames[lastModified.getMonth()] + ' ' + lastModified.getFullYear();
report_template_raw = report_template_raw.replace("{{LastModified}}", lastModifiedString);

// Make conditional substitutions

if (show_test_specs) {
  report_template_raw = report_template_raw.replace("{{TestSpecTOC}}",
    '<li class="tocline"><a href="testing/report.html#testspecsB">Test specifications</a></li>');
  report_template_raw = report_template_raw.replace("{{TestSpec}}",testspec_template_raw); 
} else {
  report_template_raw = report_template_raw.replace("{{TestSpecTOC}}","");
  report_template_raw = report_template_raw.replace("{{TestSpec}}",""); 
}

if (show_interop_results) {
  report_template_raw = report_template_raw.replace("{{InteropTOC}}",
    '<li class="tocline">8.3 <a href="testing/report.html#test_interop">Interoperability results</a></li>');
  report_template_raw = report_template_raw.replace("{{Interop}}",interop_template_raw); 
} else {
  report_template_raw = report_template_raw.replace("{{InteropTOC}}","");
  report_template_raw = report_template_raw.replace("{{Interop}}",""); 
}

var report_dom = cheerio.load(report_template_raw);

// Read in index.html store as a dom
// (Synchronous)
const src_raw = fs.readFileSync(src_htmlfile, 'UTF-8');
var src_dom = cheerio.load(src_raw);

// Check if an dom object is in an array.  Does deep comparison.
// Used to check for assertion variants/duplicates.  We can't just 
// use the built-in Set since it does not do a deep compare and there
// is no way to customize it (insert swearing here).
function is_object_in_array(o,a) {
  let t1 = o.toString();
  for (let i=0; i<a.length; i++) {
    let t2 = a[i].toString();
    if (t1 == t2) return true;
  }
  return false;
}

// push a new item onto an array only if it is unique, using above
// deep compare to check for duplicates; return new array.
// also converts undefined values to arrays with just one element.
function push_uniq(a,o) {
  if (undefined === a) return [o];
  if (!is_object_in_array(o,a)) {
    a.push(o);
  }
  return a;
}

// Extract assertions
// Assertions are generally in spans or divs and a previous version
// of this code looked for those separately.  This version looks only for the 
// class, so in theory it could be used in other tags.  However, to avoid
// conflicts with other classes it is still recommended to use separate spans/divs
// for assertions.   This code also looks for all assertion classes at once, including
// table and default assertions, rather than separately, in an attempt to preserve the order.  
// In order to avoid breaking other code, the index of each id is maintained in a separate map.
// This index is then used later to sort results.
//
// What makes indexing annoyingly complicated is the presence of child assertions, some
// of which (not all) appear in the body, but some of which are given as "extra" assertions
// in a separate input file (these break complex assertions down into more testable components).
// We'd *like* these to appear in order directly after their parent
// but the ones in the separate document aren't scanned at
// the same time.  The solution is to use indexes of the form i.x for child assertions
// with i being the index of the parent assertions and the x being between 01 and 99 (max
// of 98 child assertions).
// (Synchronous)
var assertion_index = {};
var max_child_assertion_index = {};
var max_assertion_index = 1; // start indices at 1, consistent with children starting at .01
var src_assertions = {};
var tab_assertions = {};
var def_assertions = {};
src_dom('.rfc2119-assertion, .rfc2119-table-assertion, .rfc2119-default-assertion').each(function(i,elem) {
    let id = src_dom(this).attr('id');
    let cl = src_dom(this).attr('class');
    if ("rfc2119-assertion" === cl) {
        if (undefined === id) {
            if (warn_v) console.log("WARNING: rfc2119-assertion without id:",src_dom(this).html());
        } else {
            src_assertions[id] = push_uniq(src_assertions[id],src_dom(this));
            if (id.indexOf('_') > -1) { // is this a child assertion?
                let p_id = id.substr(0,id.indexOf('_')); // find parent id
                let p_ix = assertion_index[p_id]; // find parent index
                if (undefined == p_ix) {
                    if (warn_v) console.log("WARNING: Child assertion "+a+" defined before parent:",src_dom(this).html);
                } else {
                    // find last child assertion assigned, increment it, and assign it
                    max_child_assertion_index[p_id] += 1;
                    let c_ix = p_ix + max_child_assertion_index[p_id]/100.0;
                    assertion_index[id] = c_ix;
                }
            } else {
                // new parent assertion
                assertion_index[id] = max_assertion_index;
                max_assertion_index = max_assertion_index+1;
                max_child_assertion_index[id] = 0; // for any possible future children
            }
        }
    }
    if ("rfc2119-table-assertion" === cl) {
        if (undefined === id) {
            if (warn_v) console.log("WARNING: rfc2119-table-assertion without id:", src_dom(this).html());
        } else {
            let assertion_data = src_dom(this).children('td').map(function(i, el) {
                return src_dom(this).html();
            }).get();
            console.log("TTTTABLE " + assertion_data[2]);
            let assertion = '<span class="rfc2119-table-assertion">' 
                          + assertion_data[0]         // vocab term
                          + ': ' + assertion_data[1]  // vocab text
                          + (("mandatory" === assertion_data[2]) ? '<hr/>\nMUST be included.' : '<hr/>\nMAY be included.')
                          + (("." === assertion_data[3]) ? '' : '\nType: '+assertion_data[3]+'.')
                          +'</span>';
            if (chatty_v) console.log("table assertion",id,"added");
            if (debug_v) console.log("  text:",assertion);
            tab_assertions[id] = push_uniq(tab_assertions[id],cheerio.load(assertion)("span"));
            if (id.indexOf('_') > -1) { // is this a child assertion?
                let p_id = id.substr(0,id.indexOf('_')); // find parent id
                let p_ix = assertion_index[p_id]; // find parent index
                if (undefined == p_ix) {
                    if (warn_v) console.log("WARNING: Child assertion "+a+" defined before parent:",src_dom(this).html);
                } else {
                    // find last child assertion assigned, increment it, and assign it
                    max_child_assertion_index[p_id] += 1;
                    let c_ix = p_ix + max_child_assertion_index[p_id]/100.0;
                    assertion_index[id] = c_ix;
                }
            } else {
                // new parent assertion
                assertion_index[id] = max_assertion_index;
                max_assertion_index = max_assertion_index+1;
                max_child_assertion_index[id] = 0; // for any possible future children
            }
        }
    }
    if ("rfc2119-default-assertion" === cl) {
        if (undefined === id) {
            if (warn_v) console.log("WARNING: rfc2119-default-assertion without id:", src_dom(this).html());
        } else {
            let assertion_data = src_dom(this).children('td').map(function(i, el) {
                return src_dom(this).html();
            }).get();
            let assertion = '<span class="rfc2119-default-assertion">' 
                          + 'The value associated with member '
                          + '"<code>'
                          + assertion_data[1]  // vocab term
                          + '</code>"'
                          + ' if not given MUST be assumed to have the default value ' 
                          + '"<code>' 
                          + assertion_data[2]  // default value
                          + '</code>".' 
                          +'</span>';
            let contexts = assertion_data[0];
            if (undefined !== contexts && "null" !== contexts) {
               // get rid of any markup (convert to spaces)
               contexts = contexts.replace(/<\/?[a-zA-Z]+>/gi,' ');
            }
            depends.set(id,{
              "parents": "td-serialization-default-values",
              "contexts": contexts
            });
            if (chatty_v) console.log("default assertion",id,"added");
            if (debug_v) console.log("  text:",assertion);
            def_assertions[id] = push_uniq(def_assertions[id],cheerio.load(assertion)("span"));
            if (id.indexOf('_') > -1) { // is this a child assertion?
                let p_id = id.substr(0,id.indexOf('_')); // find parent id
                let p_ix = assertion_index[p_id]; // find parent index
                if (undefined == p_ix) {
                    if (warn_v) console.log("WARNING: Child assertion "+a+" defined before parent:",src_dom(this).html);
                } else {
                    // find last child assertion assigned, increment it, and assign it
                    max_child_assertion_index[p_id] += 1;
                    let c_ix = p_ix + max_child_assertion_index[p_id]/100.0;
                    assertion_index[id] = c_ix;
                }
            } else {
                // new parent assertion
                assertion_index[id] = max_assertion_index;
                max_assertion_index = max_assertion_index+1;
                max_child_assertion_index[id] = 0; // for any possible future children
            }
        }
    }
});

// Read in extra assertions and store as a dom
// (Synchronous)
const ea_raw = fs.readFileSync(ea_htmlfile, 'UTF-8');
var ea_dom = cheerio.load(ea_raw);

// Extract extra assertions, store in map
// (Synchronous)
var extra_assertions = {};
ea_dom('.rfc2119-assertion').each(function(i,elem) {
    let id = ea_dom(this).attr('id');
    if (undefined === id) {
        if (warn_v) console.log("WARNING: rfc2119-assertion without id:", ea_dom(this).html());
    } else {
        extra_assertions[id] = push_uniq(extra_assertions[id],ea_dom(this));
        if (id.indexOf('_') > -1) { // is this a child assertion?
            let p_id = id.substr(0,id.indexOf('_')); // find parent id
            let p_ix = assertion_index[p_id]; // find parent index
            if (undefined == p_ix) {
                if (warn_v) console.log("WARNING: Child assertion "+a+" defined before parent:",ea_dom(this).html);
            } else {
                // find last child assertion assigned, increment it, and assign it
                max_child_assertion_index[p_id] += 1;
                let c_ix = p_ix + max_child_assertion_index[p_id]/100.0;
                assertion_index[id] = c_ix;
            }
        } else {
            // new parent assertion
            assertion_index[id] = max_assertion_index;
            max_assertion_index = max_assertion_index+1;
            max_child_assertion_index[id] = 0; // for any possible future children
        }
    }
});

// Utility function to format indices
// If an integer, format without decimals.  If not an integer, format with two decimal places.
// Returns a string.
function fIx(ix) {
  let rix = Math.round(ix);
  if (rix == ix) return ix.toString(); // is an integer
  return ix.toFixed(2);
}

// Get all results, convert from CSV to JSON, Merge
// Note: this only looks at ID and Status columns.  Other columns are ignored.
// (Asynchronous)
var results = new Map();
var results_files = fs.readdirSync(results_dir);
function get_results(i,done_callback) {
    if (0 == results_files.length) {
        done_callback(results);	    
    }	    
    var file = path.join(results_dir, results_files[i]);
    if (file.match(/.csv$/g)) {
        if (info_v) console.log("processing results in",file);
        var basename = path.basename(file,'.csv');
        var filedata = fs.readFileSync(file).toString();
        csvtojson()
            .fromString(filedata)
            .then((data)=> {
                results.set(basename,data);
                if (results_files.length - 1 == i) {
                    done_callback(results);
                } else {
                    get_results(i+1,done_callback);
                }
            })
    } else {
        if (results_files.length - 1 == i) {
            done_callback(results);
        } else {
            get_results(i+1,done_callback);
        }
    }
}
var merged_results = new Map();
function merge_results(done_callback) {
    for (let [impl,data] of results.entries()) {
        for (let i=0; i<data.length; i++) {
           let id = data[i]["ID"];
           let st = data[i]["Status"];
           let pass = undefined;
           if ("pass" === st) pass = 1;
           let fail = undefined;
           if ("fail" === st) fail = 1;
           let notimpl = undefined;
           if ("not-impl" === st) notimpl = 1;
           let current = merged_results.get(id);
           // there has GOT to be an easier way to do this... oh, well
           if (undefined != current) {
               // passes
               if (undefined != current.pass) {
                   if (undefined != pass) {
                       current.pass += pass;
                       merged_results.set(id,current);
                   }
               } else {
                   if (undefined != pass) {
                       current.pass = pass;
                       merged_results.set(id,current);
                   }
               }
               // failures
               if (undefined !== current.fail) {
                   if (undefined !== fail) {
                       current.fail += fail;
                       merged_results.set(id,current);
                   }
               } else {
                   if (undefined !== fail) {
                       current.fail = fail;
                       merged_results.set(id,current);
                   }
               }
               // not-implemented
               if (undefined !== current.notimpl) {
                   if (undefined !== notimpl) {
                       current.notimpl += notimpl;
                       merged_results.set(id,current);
                   }
               } else {
                   if (undefined !== notimpl) {
                       current.notimpl = notimpl;
                       merged_results.set(id,current);
                   }
               }
           } else {
               merged_results.set(id,{"pass": pass,"fail": fail,"notimpl": notimpl});
           }
        }
    }
    done_callback(merged_results);
}

// Get dependencies
// (Asynchronous)
function get_depends(done_callback) {
    if (info_v) console.log("processing dependencies in",depends_csvfile);
    var filedata = fs.readFileSync(depends_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["ID"];
                if (undefined !== id) {
                    depends.set(id,{
                        "parents": item["Parents"],
                        "contexts": item["Contexts"]
                    });
                }
                if (debug_v) console.log(id," depends ",depends.get(id));
            }
            done_callback();
        });
}

// Get categories
// (Asynchronous)
var categories = new Map();
function get_categories(done_callback) {
    if (info_v) console.log("processing categories in",categories_csvfile);
    var filedata = fs.readFileSync(categories_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["ID"];
                let cat = item["Category"];
                if (undefined !== id && undefined !== cat) {
                    categories.set(id,cat);
                    if (chatty_v) console.log("add category record for id",id+":",categories.get(id));
                } else {
                    if (warn_v) console.log("WARNING: category record for id",id,"in unexpected format");
                }
            }
            done_callback();
        });
}

// Get suppressed assertions
// (Asynchronous)
var suppressed = new Map();
function get_suppressed(done_callback) {
    if (info_v) console.log("processing suppressed assertions in",suppressed_csvfile);
    var filedata = fs.readFileSync(suppressed_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["ID"];
                let st = item["Status"];
                if (undefined !== id && undefined !== st) {
                    suppressed.set(id,st);
                    if (chatty_v) console.log("add suppressed record for id",id+":",suppressed.get(id));
                } else {
                    if (warn_v) console.log("WARNING: suppressed record for id",id,"in unexpected format");
                }
            }
            done_callback();
        });
}

// Get manual assertions
// (Asynchronous)
var manual = new Map();
function get_manual(done_callback) {
    if (info_v) console.log("processing manual assertions in",manual_csvfile);
    var filedata = fs.readFileSync(manual_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["ID"];
                let st = item["Status"];
                if (undefined !== id && undefined !== st) {
                    manual.set(id,st);
                    if (chatty_v) console.log("add manual record for id",id+":",manual.get(id));
                } else {
                    if (warn_v) console.log("WARNING: manual record for id",id,"in unexpected format");
                }
            }
            done_callback();
        });
}

// Get implementation data
// (Asynchronous)
var impls = new Map();
function get_impls(done_callback) {
    if (info_v) console.log("processing implementation data in",impls_csvfile);
    var filedata = fs.readFileSync(impls_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["Implementation"];
                let org = item["Organization"];
                let name = item["Name"];
                let roles = item["Roles"];
                if (undefined !== roles) {
                    roles = roles.split(' ');
                }
                if (undefined !== id) {
                    impls.set(id,{
                        "org": org,
                        "name": name,
                        "roles": roles
                    });
                    if (chatty_v) console.log("add impl record for id",id+":",impls.get(id));
                } else {
                    if (warn_v) console.log("WARNING: impl record for id",id,"in unexpected format");
                }
            }
            done_callback();
        });
}

// At-Risk Items
// (Asynchronous)
var risks = new Map();
function get_risks(done_callback) {
    var risks_css = "";
    if (info_v) console.log("processing risks in",atrisk_csvfile);
    var filedata = fs.readFileSync(atrisk_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let id = item["ID"];
                if (undefined !== id) {
                    risks.set(id,true);
                    risks_css += '#' + id + ' {\n'
                               + '  background-color: yellow;\n'
                               + '}\n';
                    if (chatty_v) console.log("add at-risk record for",id);
                } else {
                    if (warn_v) console.log("WARNING: at-risk record for id",id,"in unexpected format");
                }
            }
            fs.writeFileSync(atrisk_cssfile,risks_css);
            done_callback();
        });
}

// Interop Data
// (Asynchronous)
var interop = new Set();
var interop_producers = new Set();
var interop_consumers = new Set();
var interop_table = new Map();
function get_interop_file(interop_csvfile,done_callback) {
    if (info_v) console.log("processing interop data in",interop_csvfile);
    let filedata = fs.readFileSync(interop_csvfile).toString();
    csvtojson()
        .fromString(filedata)
        .then((data)=> {
            for (let i=0; i<data.length; i++) {
                let item = data[i];
                let producer = item["Producer"];
                let consumer = item["Consumer"];
                let security = item["Security"];
                let teststatus = item["Status"];
                if (undefined !== producer && 
                    undefined !== consumer &&
                    undefined !== security) {
                    interop.add({
                       "producer": producer,
                       "consumer": consumer,
                       "security": security
                    });
                    interop_producers.add(producer);
                    interop_consumers.add(consumer);
                    let entry_key = producer+"=>"+consumer;
                    let entry = interop_table.get(entry_key);
                    let new_entry = ("fail" === teststatus) 
                                    ? "Fail" : (("nosec" === security)
                                                ? "Pass" : "Secure"); 
                    if (undefined === entry) {
                        entry = new_entry; 
                        if (chatty_v) console.log("new interop record for",entry_key,"=",entry);
                    } else {
                        // Secure overrides "Pass" (but not "Fail"...)
                        if ("Secure" === entry && "Pass" === new_entry) {
                            entry = "Secure";
                        }
                        if ("Secure" === new_entry && "Pass" === entry) {
                            entry = "Secure";
                        }
                        if (chatty_v) console.log("update interop record for",entry_key,"=",entry);
                    }
                    interop_table.set(entry_key,entry);
                } else {
                    if (warn_v) console.log("WARNING: interop record in unexpected format:",item);
                }
            }
            done_callback();
        });
}
function get_interop_files(interop_files,done_callback) {
    if (debug_v) console.log("(D) interop files:",interop_files);
    if (0 == interop_files.length) {
        done_callback();	    
    }	    
    let interop_file = path.join(interop_dir,interop_files[0]); 
    if (debug_v) console.log("(D) interop file:",interop_file);
    if (interop_file.match(/.csv$/g)) {
        get_interop_file(interop_file,function() {
           if (interop_files.length > 1) {
               get_interop_files(interop_files.slice(1),done_callback);
           } else {
               done_callback();
           }
        });
    } else {
       if (interop_files.length > 1) {
           get_interop_files(interop_files.slice(1),done_callback);
       } else {
           done_callback();
       }
    }
}
function get_interops(done_callback) {
    let interop_files = fs.readdirSync(interop_dir);
    get_interop_files(interop_files,done_callback);
}

// Clear (well, write headers for) assertions table 
// (Synchronous)
fs.writeFileSync(report_csvfile,'"Index","ID","Status","Comment","Assertion"\n');

// Merge implementation descriptions
// (Asynchronous)
function merge_implementations(done_callback) {
  // insert implementation descriptions
  let i = 1;
  for (desc_id in descs) {
      let desc = descs[desc_id];
      let desc_name = desc_names[desc_id];
      report_dom('ul#systems-toc').append('<li class="tocline"></li>\n');
      let report_li = report_dom('ul#systems-toc>li:last-child');
      report_li.append('\n\t6.'+i+' <a href="'+report_base+'#'+desc_id+'">'+desc_name+'</a>');
      i++;
      report_dom('#systems-impl').append(desc);
  }
  done_callback();
}

// Merge interop table
// (Asynchronous)
function merge_interops(done_callback) {
  if (show_interop_results) {
		interop_consumers.forEach(function(consumer) {
				if (chatty_v) console.log("merge_interops consumer:",consumer);
				let report_tr = report_dom('table#testinterop>thead>tr:last-child');
				let impl = impls.get(consumer);
				let impl_org = "";
				let impl_name = "Unknown";
				if (undefined === impl) {
					 if (warn_v) console.log("no name available for impl",item);
				} else {
					 impl_org = impl.org;
					 impl_name = impl.name;
				}
				report_tr.append('\n\t<th class="rotate" id="'
												 +consumer
												 +'"><div><span>'
												 +impl_org
												 +'</span><br/><span>'
												 +impl_name
												 +'</span></div></th>\n');
		});
		interop_producers.forEach(function(producer) { 
				let impl = impls.get(producer);
				let impl_org = "";
				let impl_name = "Unknown";
				if (undefined === impl) {
					 if (warn_v) console.log("no name available for impl",producer);
				} else {
					 impl_org = impl.org;
					 impl_name = impl.name;
				}
				let report_tr = report_dom('table#testinterop>tbody:last-child');
				report_tr.append('\n<tr><th id="'
												 +producer
												 +'">'
												 + impl_org
												 + '<br/>'
												 + impl_name
												 +'</th></tr>');
				interop_consumers.forEach(function(consumer) {
						let pair_id = producer+"=>"+consumer;
						let item_status = interop_table.get(pair_id);
						let report_elem = report_dom('table#testinterop>tbody>tr:last-child');
						if (undefined === item_status) {
								report_elem.append('\n\t<td class="notimpl">Not Impl</td>');
						} else if ("Secure" === item_status) {
								report_elem.append('\n\t<td class="secure">'+item_status+'</td>');
						} else if ("Pass" === item_status) {
								report_elem.append('\n\t<td class="passed">'+item_status+'</td>');
						} else {
								report_elem.append('\n\t<td class="failed">'+item_status+'</td>');
						}
				});
		});
  }
  done_callback();
}

// Merge assertions into a single array
// (Asynchronous)
var assertion_array = [];

function merge_assertions(assertions,ac,done_callback) {
  // insert assertions
  for (a_id in assertions) {
    let n = assertions[a_id].length;
    for (let i = 0; i < n; i++) { 
      a_dom = assertions[a_id][i];
      let assertion_object = {};
      let a = a_id;
      if (n > 1) a += "-" + (i+1);
      if (chatty_v) console.log("Processing assertion "+a);

      assertion_object.ix = assertion_index[a];
      assertion_object.id = a;
      assertion_object.ac = ac;
      assertion_object.text = a_dom.text();
 
      assertion_array.push(assertion_object);
    }
  }
  done_callback();
}

// Scan assertions to identify children; clear parents
// of such child assertions, then rescan and assign to 
// parent the minimum pass and maximum fails of any child.
function process_children(done_callback) {
  // Find set of all parents
  let n = assertion_array.length;
  let ps = new Set();
  for (let i = 0; i < n; i++) { 
    a = assertion_array[i].id;
    if (undefined === suppressed.get(a)) { 
      if (a.indexOf('_') > -1) {
        let p = a.substr(0,a.indexOf('_'));
        ps.add(p);
      }
    }
  }
  console.log(ps);
  // for every parent, clear data in merged assertions 
  ps.forEach( p => {
    merged_results.set(p,undefined);
  });
  // rescan children, rederive parent's scores
  for (let i = 0; i < n; i++) { 
    a = assertion_array[i].id;
    if (undefined === suppressed.get(a)) { 
     if (a.indexOf('_') > -1) {
      let p = a.substr(0,a.indexOf('_'));
      let rp = merged_results.get(p);
      let rc = merged_results.get(a);
      if (undefined === rp) {
        if (undefined !== rc) merged_results.set(p,rc);
      } else {
        if (undefined !== rc) {
          let rp_pass = (undefined === rp.pass) ? 0 : rp.pass;
          let rp_fail = (undefined === rp.fail) ? 0 : rp.fail;
          let rp_notimpl = (undefined === rp.notimpl) ? 0 : rp.notimpl;
          let rc_pass = (undefined === rc.pass) ? 0 : rc.pass;
          let rc_fail = (undefined === rc.fail) ? 0 : rc.fail;
          let rc_notimpl = (undefined === rc.notimpl) ? 0 : rc.notimpl;
          merged_results.set(p,{
            "pass": Math.min(rp_pass, rc_pass),
            "fail": Math.max(rp_fail, rc_fail),
            "notimpl": Math.max(rp_notimpl, rc_notimpl)
          });
        }
      }
     }
    }
  }
  // next
  done_callback();
}

function format_assertions(done_callback) {
  assertion_array.sort((a,b) => {
    return (a.ix < b.ix) ? -1 : ((a.ix > b.ix) ? 1 : 0); 
  });
  let n = assertion_array.length;
  for (let i = 0; i < n; i++) { 
    let ix = assertion_array[i].ix;
    let a = assertion_array[i].id;
    let ac = assertion_array[i].ac;
    let a_text = assertion_array[i].text;
    a_text = a_text.replace(/[\r\n]/gm,' ').trim(); // clean up newlines 
    a_text = a_text.replace(/[ \t]+/gm,' '); // replace tabs and multiple spaces with one space
    a_text = a_text.trim(); // trim white space at start and end
    let qa_text = a_text.replace(/"/gm,'""'); // escape quotes (for CSV)

    // Determine if assertion is suppressed
    if (undefined !== suppressed.get(a)) { 
      if (chatty_v) console.log("Suppressing assertion "+a);
    } else {
      // Determine if is manual assertion
      let ma = (undefined !== manual.get(a));
      let tid = (ma ? "manualresults" : "testresults");

      if (chatty_v) console.log("Formatting assertion "+a);

      // Assertions table
      fs.appendFileSync(report_csvfile, fIx(ix)+',"'+a+'","null",,"'+qa_text+'"\n');

      if (show_test_specs) {
        // Test Specifications Appendix
        report_dom('#testspecs').append('\n<dt></dt>');
        let report_dt = report_dom('#testspecs>dt:last-child');
        report_dt.append('\n\t<a href="'+report_base+'#'+a+'">'+fIx(ix)+': '+a+'</a>:');
        if ("tabassertion" === ac) {
          report_dt.append(' <em>(table)</em>');
        }
        if ("extraassertion" === ac) {
          report_dt.append(' <em>(extra)</em>');
        }
      }

      let category = undefined;
      let req = false;
      {
        if (a_text.indexOf('MUST') > -1) {
          if (a_text.indexOf('MUST NOT') > -1) {
            category = 'MUST NOT';
          } else {
            category = 'MUST';
          }
          req = true;
        }
        if (a_text.indexOf('SHOULD') > -1) {
          if (a_text.indexOf('SHOULD NOT') > -1) {
            category = 'SHOULD NOT';
          } else {
            category = 'SHOULD';
          }
        }
        if (a_text.indexOf('MAY') > -1) {
          category = 'MAY';
        }
        if (a_text.indexOf('REQUIRED') > -1) {
          category = 'REQUIRED';
          req = true;
        }
        if (a_text.indexOf('RECOMMENDED') > -1) {
          category = 'RECOMMENDED';
        }
        if (a_text.indexOf('OPTIONAL') > -1) {
          category = 'OPTIONAL';
        }

        if (undefined === category) {
          if (warn_v) console.log("WARNING: RFC2119 category is not defined for",a);
        }
        if (show_test_specs) {
          if (repeat_assertion_in_appendix) {
            if (undefined === category) {
              report_dt.append(': <strong>category is undefined</strong>');
            } else {
              report_dt.append(': <strong>'+category+'</strong>');
            }
          }
        }
      }
  
      // Make table row
      let d = depends.get(a);
      report_dom('table#'+tid+'>tbody:last-child')
        .append('\n<tr id="'+a+'" class="'+ac+'"></tr>');
      let report_tr = report_dom('tr#'+a);

      // Generated subassertions still must link to the actual assertion in the TD spec
      let a_frag = a;
      if (a_frag.indexOf("_") !== -1) {
        a_frag = a_frag.split("_")[0];
      }

      // ID
      report_tr.append('\n\t<td class="'+ac+'"><a target="spec" href="'+src_base+'#'+a_frag+'">'+fIx(ix)+': '+a+'</a></td>');

      // Category
      if (undefined != categories.get(a)) {
        report_tr.append('\n\t<td class="'+ac+'">'+categories.get(a)+'</td>\n');
      } else {
        report_tr.append('\n\t<td class="'+ac+'"></td>');
      }

      // Required pass threshold
      let pass_threshold;
      if (req) {
        report_tr.append('\n\t<td class="'+ac+'">Y</td>');
        pass_threshold = 2;
      } else {
        report_tr.append('\n\t<td class="'+ac+'">N</td>');
        // pass_threshold = 1;
        pass_threshold = 2;
      }

      // Context(s)
      if (undefined != d) {
        let c = d.contexts;
        if (undefined != c && "null" !== c) {
          let cs = c.split(' ');
          let h = '\n\t<td class="'+ac+'">';
          for (let i=0; i<cs.length; i++) {
            if (0 != cs[i].trim().length) {
              if (cs[i].trim().startsWith("(")) {
                // no link for higher-level contexts such as TD Producers
                h = h + '\n\t\t' + cs[i] + '<br/>';
              } else {
                // link in column 4 to TD spec context -- must point to TD spec, where fragments are lower case
                h = h + '\n\t\t<a target="spec" href="'+src_base+'#' + cs[i].toLowerCase() + '">' + cs[i] + '</a><br/>';
              }
            }
          }
          report_tr.append(h+'\n\t</td>');
        } else {
          report_tr.append('\n\t<td class="'+ac+'"></td>');
        }
      } else {
        report_tr.append('\n\t<td class="'+ac+'"></td>');
      }

      // Assertion
      if (undefined != risks.get(a)) {
        report_tr.append('\n\t<td class="atrisk">'+a_text+'</td>');
      } else {
        report_tr.append('\n\t<td class="'+ac+'">'+a_text+'</td>');
      }

      // Parent(s)
      if (undefined != d) {
        let p = d.parents;
        if (undefined != p && "null" !== p) {
          // mk: I cannot see how there should be multiple parents and there is no such case currently
          let ps = p.split(' ');
          let h = '\n\t<td class="'+ac+'">';
          for (let i=0; i<ps.length; i++) {
            if (0 != ps[i].trim().length) {
              // link in column 6 to base assertion
              h = h + '\n\t\t<a href="'+report_base+'#' + ps[i] + '">' + ps[i] + '</a><br>';
            }
          }
          report_tr.append(h+'\n\t</td>');
        } else {
          report_tr.append('\n\t<td class="'+ac+'"></td>');
        }
      } else {
        report_tr.append('\n\t<td class="'+ac+'"></td>');
      }

      // Test Results
      let result = merged_results.get(a);
      if (undefined != result) {
        // Number of reported pass statuses
        let pass = result.pass;
        if (undefined != pass) {
          if (pass >= pass_threshold) {
            report_tr.append('\n\t<td class="result '+ac+'">'+pass+'</td>');
          } else {
            if (pass == 0) {
              report_tr.append('\n\t<td class="result missing">'+pass+'</td>');
            } else {
              report_tr.append('\n\t<td class="result failed">'+pass+'</td>');
            }
          }
        } else {
          report_tr.append('\n\t<td class="result missing">0</td>');
          pass = 0;
        }
        // Number of reported fail statuses
        let fail = result.fail;
        if (undefined != fail) {
          if (fail > 0) {
            report_tr.append('\n\t<td class="result failed">'+fail+'</td>');
          } else {
            report_tr.append('\n\t<td class="result '+ac+'">'+fail+'</td>');
          }
        } else {
          report_tr.append('\n\t<td class="result '+ac+'">0</td>');
          fail = 0;
        }
        // Number of reported not implemented statuses
        let notimpl = result.notimpl;
        if (undefined != notimpl) {
          report_tr.append('\n\t<td class="result '+ac+'">'+notimpl+'</td>');
        } else {
          report_tr.append('\n\t<td class="result '+ac+'">0</td>');
          notimpl = 0;
        }
        // Total number of reported statuses
        let totals = pass + fail + notimpl;
        if (0 == totals) {
          report_tr.append('\n\t<td class="result missing">0</td>');
        } else if (totals < 2) {
          report_tr.append('\n\t<td class="result failed">'+totals+'</td>');
        } else {
          report_tr.append('\n\t<td class="result '+ac+'">'+totals+'</td>');
        }
      } else {
        report_tr.append('\n\t<td class="result missing">0</td>');
        report_tr.append('\n\t<td class="result missing">0</td>');
        report_tr.append('\n\t<td class="result missing">0</td>');
        report_tr.append('\n\t<td class="result missing">0</td>');
      }

      // Add to test spec appendix
      if (show_test_specs) {
        report_dom('#testspecs').append('\n<dd id="'+a+'" class="'+ac+'"></dd>');
        let report_dd = report_dom('dd#'+a);
        if (repeat_assertion_in_appendix) {
          report_dd.append(a_text);
          report_dd.append('\n\t<br/>');
        }
        a_spec = testspec[a];
        report_dd.append('\n\t<span></span>');
        let report_li = report_dom('dd#'+a+'>span:last-child');
        if (undefined === a_spec) {
          if (warn_v) console.log("WARNING: no test spec for",a);
          report_li.append('\n\t\t<p><strong>NO TEST SPECIFICATION</strong></p>');
        } else {
          report_li.append('\n\t\t'+a_spec);
        }
      }
    }
  }
  done_callback(false); // success
}

function output_atrisk(done_callback) {
  let stream = fs.createWriteStream(atrisk_out_csvfile,{flags:'w'});
  stream.write('"Index","ID","Fix"\n');
  let n = assertion_array.length;
  for (let i = 0; i < n; i++) { 
    let ix = assertion_array[i].ix;
    let a = assertion_array[i].id;
    let ac = assertion_array[i].ac;
    let a_text = assertion_array[i].text;

    // Determine if assertion is suppressed
    if (undefined !== suppressed.get(a)) { 
      if (chatty_v) console.log("Suppressing assertion "+a);
    } else {
      if (chatty_v) console.log("Analysing assertion "+a);

      let category = undefined;
      let req = false;
      if (a_text.indexOf('MUST') > -1) req = true;
      if (a_text.indexOf('REQUIRED') > -1) req = true;

      // get depends
      let d = depends.get(a);

      // Compute required threshold
      let pass_threshold;
      if (req) {
        pass_threshold = 2;
      } else {
        // pass_threshold = 1;
        pass_threshold = 2;
      }

      // Test Results
      let result = merged_results.get(a);
      if (undefined === result) {
        stream.write(fIx(ix)+',"'+a+'","HELP, no results"\n');
      } else {
        // Check number of reported pass statuses
        let pass = result.pass;
        if (undefined === pass) {
          stream.write(fIx(ix)+',"'+a+'","HELP, pass undefined"\n');
        } else {
          if (pass < pass_threshold) {
            stream.write(fIx(ix)+',"'+a+'","HELP, pass='+pass+'"\n');
          }
        }
      }
    }
  }
  stream.end();
  done_callback(false); // success
}

get_results(0,function(results) {
    if (debug_v) {
        console.log("Results: {\n");
        for (let [key,data] of results.entries()) {
            console.log(key," => ",data);
        }
        console.log("}");
    }
    merge_results(function(merged_results) {
    if (debug_v) {
      console.log("Merged Results: {\n");
      for (let [key,data] of merged_results.entries()) {
        console.log(key," => ",data);
      }
      console.log("}");
    }
    get_risks(function() {
     get_depends(function() {
      get_categories(function() {
       if (chatty_v) console.log("categories: ",categories);
       get_manual(function() {
        if (chatty_v) console.log("manual: ",manual);
        get_suppressed(function() {
         if (chatty_v) console.log("suppressed: ",suppressed);
         get_impls(function() {
          if (chatty_v) console.log("impls: ",impls);
          get_interops(function() {
           if (chatty_v) {
            console.log("interop: ",interop);
            console.log("interop producers: ",interop_producers);
            console.log("interop consumers: ",interop_consumers);
            console.log("interop table: ",interop_table);
           }
           merge_implementations(function() {
            merge_interops(function() {
             merge_assertions(src_assertions,"baseassertion",function() {
              merge_assertions(tab_assertions,"tabassertion",function() {
               merge_assertions(def_assertions,"defassertion",function() {
                merge_assertions(extra_assertions,"extraassertion",function() {
                 process_children(function() {
                  format_assertions(function() {
                   // Output report
                   fs.writeFile(report_htmlfile, report_dom.html(), function(error) {
                    if (error) {
                     return console.log(err);
                    } else {
                     if (info_v) console.log("Report output to "+report_htmlfile);
                     // Output at-risk items
                     output_atrisk(function(error) {
                      if (error) {
                       return console.log(err);
                      } else {
                       if (info_v) console.log("Flagged atrisk assertions output to "+atrisk_out_csvfile);
                      }
                     }); 
                    }
                   }); 
                  }); 
                 }); 
                });
               });
              }); 
             }); 
            }); 
           }); 
          }); 
         }); 
        }); 
       }); 
      }); 
     });
    });
  });
});