Changeset 14502 in main


Ignore:
Timestamp:
08/22/17 22:39:28 (5 years ago)
Author:
Garth Braithwaite
Message:

java, proxy - added the proxy webapp and add missing values updates.

Location:
trunk
Files:
24 added
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/ibisph/src/main/java/org/ibisph/querymodule/modelmap/AddMissingQueryResultRecordsToQueryModule.java

    r14457 r14502  
    22
    33import java.util.ArrayList;
    4 import java.util.LinkedHashMap;
     4import java.util.HashMap;
    55import java.util.List;
    66import java.util.Map;
    … …  
    1616
    1717import org.ibisph.modelmap.ProcessModelMap;
    18 import org.ibisph.querymodule.service.ArrayIndexIncrementer;
     18import org.ibisph.util.StrLib;
    1919import org.ibisph.util.XMLLib;
    2020
     21
    2122/**
    22  * Adds the missing value dimension records to the supplied query module model
    23  * if the QM's configuration has the ADD_MISSING_QUERY_RESULT_RECORDS struct
    24  * defined.  This provides a similar effect as using the
    25  * QUERY_APPLICATION_REQUEST_ADD_NON_FILTERED_GROUP_BY_DIMENSION_VALUES_FLAG.
    26  * The IBISQ processing is typically a better solution as SAS can do additional
    27  * checks on the data and provide more approp data values.  This feature (if
    28  * wired up) has is useful for survey type data or if an adopter needs to use
    29  * on some data that is not supported via the backend IBISQ/SAS data frame etc.
    30  *
    31  * NOTE: This modelmap update code will ONLY supplement missing value type
    32  * records.  It does not create records if no records are returned from the
    33  * query. 
     23 * Adds the missing dimension value records to the supplied query module's
     24 * IBISQ request result records under these conditions:
     25 * A) The QM's configuration has the ADD_MISSING_QUERY_RESULT_RECORDS struct
     26 *    defined (wired up).  This provides a similar effect as using the
     27 *    QUERY_APPLICATION_REQUEST_ADD_NON_FILTERED_GROUP_BY_DIMENSION_VALUES_FLAG.
     28 *    The IBISQ processing is typically a better solution as SAS can do
     29 *    additional checks on the data and provide more approp data values.  This
     30 *    feature (if wired up) is useful for survey type data or if an adopter
     31 *    needs to use on some data that is not supported via the backend IBISQ/SAS
     32 *    data frame etc.
     33 * B) The specified, associated DIM VALUES (via the DIMENSION_NAME or the
     34 *    ALL_FLAG) are NOT filtered.
     35 * C) Does NOT create/supplement missing value type records if no records.
     36 *
     37 * IMPORTANT: This modelmap update code MUST be ran after the IBISQ query result
     38 * records are inserted into the QM XML doc.
     39 *
     40 * SECURITY NOTE: Don't need to worry about non user auth values as these are
     41 * a) blank, and b) auto filtered by the security related code.
    3442 *
    3543 * STRUCTURE:
    … …  
    7381
    7482    Node queryModuleConfigurationNode = XMLLib.getNode(
    75         queryModule, 
     83        queryModule,
    7684        "/QUERY_MODULE/CONFIGURATIONS/CONFIGURATION[NAME = /QUERY_MODULE/REQUEST/CONFIGURATION_NAME]"
    7785    );
    … …  
    8189      throw new Exception("Requested query module configuration [" + configurationName + "] does NOT exist in query module: " + queryModuleName);
    8290    }
     91
     92    // if no missing values specified for this config return.
    8393    Node addMissingResultRecordsNode = XMLLib.getNode(
    8494        queryModuleConfigurationNode,
    8595        "ADD_MISSING_QUERY_RESULT_RECORDS"
    8696    );
    87 
    88     // If add missing values element exists then proceed. 
     97    if(null == addMissingResultRecordsNode) return;
     98
     99    // if no records return;
     100    Node queryResultRecords = XMLLib.getNode(
     101        queryModule,
     102        "/QUERY_MODULE/IBISQ_QUERY_RESULT/RECORDS/RECORD"
     103    );
     104    if(null == queryResultRecords) {
     105      logger.debug(".addMissingQueryResultRecordElements - no result records NOT able to process for missing values - model NOT modified.");
     106      return;
     107    }
     108
     109    // Add missing values element exists - proceed. 
    89110    // First check for the ALL_FLAG.  If add all does not exists then get and
    90111    // put all the DIMENSION_NAME elements into a list.  The sub processing
    91112    // code uses the list to determine if process all (null list) or to check
    92113    // for against the specific dimension names specified (to be matched -
    93     // deals with actual and proxy dimension names).  If not add all or any
    94     // dimension names then missing values is not processed.
    95     if(null != addMissingResultRecordsNode) {
    96       List<String> missingRecordsDimensionNameList = null;
    97       Node addAllFlagNode = XMLLib.getNode(addMissingResultRecordsNode, "ALL_FLAG");
    98       if(null == addAllFlagNode) {
    99         List<Node> missingRecordsDimensionNamesElements = XMLLib.getNodes(addMissingResultRecordsNode, ".//DIMENSION_NAME");
    100         missingRecordsDimensionNameList = new ArrayList<String>();
    101         for(int i=0; i<missingRecordsDimensionNamesElements.size(); i++) {
    102           String dimensionName = missingRecordsDimensionNamesElements.get(i).getText().trim();
    103           missingRecordsDimensionNameList.add(dimensionName);
     114    // deals with actual and proxy dimension names).  If not add all or no dim
     115    // names specified then missing values is not processed - exit.
     116    List<String> missingRecordsDimensionNameList = null;
     117    Node addAllFlagNode = XMLLib.getNode(addMissingResultRecordsNode, "ALL_FLAG");
     118    if(null == addAllFlagNode) {
     119      List<Node> missingRecordsDimensionNamesElements = XMLLib.getNodes(addMissingResultRecordsNode, ".//DIMENSION_NAME");
     120      missingRecordsDimensionNameList = new ArrayList<String>();
     121      for(int i=0; i<missingRecordsDimensionNamesElements.size(); i++) {
     122        String dimensionName = missingRecordsDimensionNamesElements.get(i).getText().trim();
     123        missingRecordsDimensionNameList.add(dimensionName);
     124      }
     125    }
     126    if((null == addAllFlagNode) && 
     127       ((null == missingRecordsDimensionNameList) || (0 == missingRecordsDimensionNameList.size()))
     128    ) return;
     129
     130    // null missingRecordsDimensionNameList == add all.
     131    // If no dimensions matches then return - don't process.
     132    List<String> actualDimensionNameList = getRequestedActualDimensionNames(queryModule);
     133    Map<String, List<String>> dimensionNameValuesMap = getDimensionValuesMap(
     134      queryModule,
     135      actualDimensionNameList,
     136      missingRecordsDimensionNameList
     137    );
     138    if(null == dimensionNameValuesMap) {
     139      logger.debug(".addMissingQueryResultRecordElements - result did not contain any dimensions that match the criteria - model not modified.");
     140      return;
     141    }
     142
     143    // if here then the dim name matched the criteria so process for missing values.
     144    logger.debug(".addMissingQueryResultRecordElements - processing for missing records - dimension map size: {}", dimensionNameValuesMap.size());
     145    Node measure = XMLLib.getNode(queryModuleConfigurationNode, ".//MEASURE[1]");
     146    String measuresName        = XMLLib.getText(measure, "NAME");
     147    String missingMeasureValue = XMLLib.getText(measure, "MISSING_QUERY_RESULT_RECORD_VALUE");
     148    if(null == missingMeasureValue) missingMeasureValue = this.defaultMissingValue;
     149    Node ancillaryValueNames = XMLLib.getNode(queryModuleConfigurationNode, "ANCILLARY_VALUE_NAMES");
     150    addMissingResultRecordElements(
     151      queryModule,
     152      dimensionNameValuesMap,
     153      measuresName,
     154      missingMeasureValue,
     155      ancillaryValueNames
     156    );
     157  } //-------------------------- End of Method ------------------------------
     158
     159
     160  /**
     161   * Builds a list of all dimension values keyed by a dimension name to be used
     162   * for checking and creating missing query result RECORD elements.  If the
     163   * missingValuesDimensionNameList is null then ALL dimension values are tested
     164   * and included (unless filtered).  If a dimension is filtered (has a selected
     165   * dimension) then that dimension is used only for xpath record lookup.  That
     166   * filtered dimension is not included as part of the "add missing record" dim
     167   * cube/determination rule. 
     168   *
     169   * This code also handles if the dim list name is a
     170   * proxy to the actual query result.
     171   * @param queryModuleDocument
     172   * @param missingValuesDimensionNameList optional list of dimension names to
     173   *   be processed.  If null then all dim names and values will be built.
     174   * @return map of all dimension name/values IF criteria is matched otherwise
     175   *   a null map is returned - which is a flag to not process.
     176   */
     177  protected Map<String, List<String>> getDimensionValuesMap (
     178      Node queryModuleDocument,
     179      List<String> actualDimensionNameList,
     180      List<String> missingValuesDimensionNameList
     181  ) {
     182    boolean addAllDimensions = (null == missingValuesDimensionNameList);
     183
     184    // make sure that the dimension names are needing to be checked.  Start
     185    // with the actual dims and if no match get that dims proxy and test it.
     186    // if no matches AND not the ALL dimensions then no add missing processing
     187    // is needed - return a null map which tells the caller nothing to add.
     188    ArrayList<String> dimensionNamesToBeUsedForMissingValues = new ArrayList<String>();
     189    for(String dimensionName : actualDimensionNameList) {
     190      if(addAllDimensions || missingValuesDimensionNameList.contains(dimensionName)) {
     191        dimensionNamesToBeUsedForMissingValues.add(dimensionName);
     192      }
     193      else {
     194        String proxyDimensionName = XMLLib.getText(
     195          queryModuleDocument,
     196          "/QUERY_MODULE/DIMENSIONS/DIMENSION[NAME='" + dimensionName + "']/PROXY_DIMENSION_NAME",
     197          ""
     198        );
     199        if(missingValuesDimensionNameList.contains(proxyDimensionName)) {
     200          dimensionNamesToBeUsedForMissingValues.add(dimensionName);
    104201        }
    105202      }
    106 
    107       // Process for missing values.  To proceed either need the all flag set
    108       // or some dimension names MUST be specified. 
    109       if((null != addAllFlagNode) || (0 < missingRecordsDimensionNameList.size())) {
    110 
    111         // Next get all the valid dim names and their values.  This map is
    112         // built based on the exiting query result record set and the missing
    113         // values list.  If the dim name list doesn't match anything then
    114         // the map returned is null which acts as a flag to not process for
    115         // any missing value.
    116         Map<String, List<String>> dimensionNameValuesMap = getDimensionValuesMap(
    117           queryModule,
    118           missingRecordsDimensionNameList
    119         );
    120 
    121         // If result does not match the criteria then return - don't process.
    122         if(null == dimensionNameValuesMap) {
    123           logger.debug(".addMissingQueryResultRecordElements - result did not contain any dimensions that match the criteria - model not modified.");
    124           return;
    125         }
    126         if(0 == dimensionNameValuesMap.size()) {
    127           logger.debug(".addMissingQueryResultRecordElements - no result dimensions/records - not adding missing values so user will see Query Too Specific message.");
    128           return;
    129         }
    130 
    131         // if here then the dim name matched the criteria so process for missing values.
    132         logger.debug(".addMissingQueryResultRecordElements - processing for missing records - dimension map size: {}", dimensionNameValuesMap.size());
    133         buildQueryResultRecordElements(
    134           queryModuleConfigurationNode,
    135           dimensionNameValuesMap
    136         );
    137       }
    138     }
    139 
    140   } //-------------------------- End of Method ------------------------------
     203    }
     204    if(dimensionNamesToBeUsedForMissingValues.isEmpty()) {
     205      logger.debug(".getDimensionValuesMap, no dimensions matched - exiting.");
     206      return(null);
     207    }
     208
     209    // If here then ALL or some dimension matched.  Build a map with the filtered
     210    // dimension values.  If the filter map size matches the dimension name
     211    // size then everything is filtered.  Return a null map as nothing needs
     212    // to be checked for/added.
     213    HashMap<String, List<String>> dimensionValuesMap = new HashMap<String, List<String>>();
     214    boolean allFiltered = true;
     215    for(String dimensionName : actualDimensionNameList) {
     216      List<String> dimensionValues = getFilteredDimensionValues(queryModuleDocument, dimensionName);
     217      if(0 == dimensionValues.size()) {
     218        allFiltered = false;
     219      }
     220      else {
     221        dimensionNamesToBeUsedForMissingValues.remove(dimensionName);
     222        dimensionValuesMap.put(dimensionName, dimensionValues);
     223      }
     224    }
     225    if(dimensionNamesToBeUsedForMissingValues.isEmpty() || allFiltered) {
     226      logger.debug(".getDimensionValuesMap, all dimensions filtered - exiting.");
     227      return(null);
     228    }
     229     
     230    // lastly if here then some dimension values need to be added.  Loop the
     231    // dimension names and if not in the map as a filtered dimension value list
     232    // then add all of the dimension's values and return the map.
     233    for(String dimensionName : actualDimensionNameList) {
     234      List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
     235      if((null == dimensionValues) || (0 == dimensionValues.size())) {
     236        dimensionValues = getAllDimensionValues(queryModuleDocument, dimensionName);
     237        dimensionValuesMap.put(dimensionName, dimensionValues);
     238      }
     239    }
     240    return(dimensionValuesMap);
     241  } //-------------------------- End of Method ------------------------------
     242
     243
     244  protected List<String> getRequestedActualDimensionNames(Node queryModuleDocument) {   
     245    ArrayList<String> dimensionNamesList = new ArrayList<String>();
     246    List<Node> dimensionNamesNodeList = XMLLib.getNodes(queryModuleDocument, "/QUERY_MODULE/REQUEST/ACTUAL_GROUP_BY/*");
     247    if(null != dimensionNamesNodeList) {
     248      for(int j=0; j<dimensionNamesNodeList.size(); j++) {
     249        String dimensionName = dimensionNamesNodeList.get(j).getText().trim();
     250        dimensionNamesList.add(dimensionName);
     251      }
     252    }
     253    return(dimensionNamesList);
     254  } //-------------------------- End of Method ------------------------------
     255
     256  protected List<String> getFilteredDimensionValues(Node queryModuleDocument, String dimensionName) {   
     257    ArrayList<String> dimensionValuesList = null;
     258 
     259    List<Node> dimensionValuesNodeList = XMLLib.getNodes(
     260      queryModuleDocument,
     261      "/QUERY_MODULE/CRITERIA/SECTIONS/SECTION/SELECTED_DIMENSIONS/SELECTED_DIMENSION[NAME = '" + dimensionName + "']/VALUES/VALUE"
     262    );
     263    if(null != dimensionValuesNodeList) {
     264      dimensionValuesList = new ArrayList<String>();
     265      for(int j=0; j<dimensionValuesNodeList.size(); j++) {
     266        String dimensionValue = dimensionValuesNodeList.get(j).getText().trim();
     267        dimensionValuesList.add(dimensionValue);
     268      }
     269    }
     270    return(dimensionValuesList);
     271  } //-------------------------- End of Method ------------------------------
     272
     273  protected List<String> getAllDimensionValues(Node queryModuleDocument, String dimensionName) {   
     274    ArrayList<String> dimensionValuesList = null;
     275    List<Node> dimensionValuesNodeList = XMLLib.getNodes(
     276      queryModuleDocument,
     277      "/QUERY_MODULE/DIMENSIONS/DIMENSION[NAME = '" + dimensionName + "']/VALUES/VALUE[not(NOT_SELECTABLE_FLAG)]"
     278    );
     279    if(null != dimensionValuesNodeList) {
     280      dimensionValuesList = new ArrayList<String>();
     281      for(int j=0; j<dimensionValuesNodeList.size(); j++) {
     282        String dimensionValue = dimensionValuesNodeList.get(j).getText().trim();
     283        dimensionValuesList.add(dimensionValue);
     284      }
     285    }
     286    return(dimensionValuesList);
     287  } //-------------------------- End of Method ------------------------------
     288
    141289
    142290
    143291  /**
    144    * Main service method that loops through the supplied document and adds
    145    * the IBIS-Q result XML elements for dimension/measure values that are
    146    * missing from the complete set of possible dimension values.
    147    * @param queryModuleConfigurationNode selected query module configuration element.
     292   * Loops through all the dimensions/values and locates the matching IBIS-Q
     293   * result record.  If record not found then a new RECORD element is created
     294   * and added.
     295   * @param queryModuleDocument selected query module configuration element.
    148296   * @param missingValuesDimensionNameList optional list of dimension names
    149297   *   to be processed.  If null then ALL dimensions are processed.
    150298   */
    151   public void buildQueryResultRecordElements(
    152       Node queryModuleConfigurationNode,
    153       Map<String, List<String>> dimensionNameValuesMap
     299  protected void addMissingResultRecordElements(
     300    Node queryModuleDocument,
     301    Map<String, List<String>> dimensionNameValuesMap,
     302    String measureName,
     303    String missingMeasureValue,
     304    Node ancillaryValueNames
    154305  ) {
    155306
    156     // Before we start building value records (with approp missing values) we
    157     // need to init some xpath related objects.  Due to issues with the Dom4j
    158     // xpath processing, we need to convert to a standard DOM and use a non
    159     // dom4j xpath lookup to test for the missing dimension value. 
    160     // ISSUE: some weird size/positional DOM4J tree issue was experienced when
    161     // doing a missing values for InfMort.  If 2009 and 1990 is selected then
     307    // Init w3c xpath related objects.  Due to issues with the Dom4j xpath
     308    // processing, we need to convert to a standard DOM and use a non dom4j
     309    // xpath lookup to test for the missing dimension value. 
     310    //
     311    // OBSERVED ISSUES:
     312    // 1) some weird size/positional DOM4J tree issue was experienced when
     313    // missing values for UT InfMort.  If 2009 and 1990 is selected then
    162314    // configured for GeoSarea to be missing and all SAs selected and grouped by
    163315    // one of the IBIS-Q RECORD elements would not be found via xpath.  If the
    … …  
    172324    // without any diff. 
    173325    // Test URL: http://localhost/ibisph-view/query/result/infmort/InfMortMainSarea/InfMortRate.html
    174     // Committed several versions of this code to the repo 6/4/2011 svn: 2784.
     326    // 2) For the HI prams module using the w3c doc and xpath some records would
     327    // be removed.  This was intermittent and was based on the XML doc and weird
     328    // combinations of filtering and grouping.  The IBISQ XML result was correct
     329    // but XPATH processing resulted in strange output.  The issue could never
     330    // be replicated in a dev enviro but was repeatable in the HI production.
     331    //
     332    // Committed several versions of this code to the repo 6/4/2011 svn: 2784.
     333    // and then the orig w3c code prior to 8/2017 commit of this code.
    175334    org.w3c.dom.Document w3cQueryModuleDocument = null;
    176335    try {
    177       w3cQueryModuleDocument = XMLLib.getW3CDocument(queryModuleConfigurationNode.getDocument());
     336      w3cQueryModuleDocument = XMLLib.getW3CDocument(queryModuleDocument.getDocument());
    178337    }
    179338    catch(TransformerException te) {
    … …  
    184343    XPathFactory factory = new net.sf.saxon.xpath.XPathFactoryImpl();
    185344    XPath xpath = factory.newXPath();
    186 
    187     // Now add/populate any missing RECORD elements if not found in the query
    188     // results XML/doc.  This is done by looping through all of the available
    189     // dimension names and assoc values and querying for that record.  If the
    190     // record is NOT found then create a new one and add it. 
    191 
    192     // setup the structs that provides all possible dimension name combinations
    193     // to be looped.
    194     int dimensionCount = dimensionNameValuesMap.size();
    195     int[]    maxIndexSize  = new int   [dimensionCount];
    196     String[] dimensionName = new String[dimensionCount];
    197     dimensionCount = 0;
    198     for(String name : dimensionNameValuesMap.keySet()) {
    199       maxIndexSize [dimensionCount] = dimensionNameValuesMap.get(name).size();
    200       dimensionName[dimensionCount] = name;
    201       dimensionCount++;
    202     }
    203     ArrayIndexIncrementer arrayIndexIncrementer = new ArrayIndexIncrementer(maxIndexSize);
    204     int[] currentDimensionIndexes = arrayIndexIncrementer.getCurrentIndex();
    205345
    206346    // loop through all the possible dimension values.  To try and preserve
    … …  
    210350    // to the new records element. Else not found, add the newly created
    211351    // missing value element to the RECORDS container.
    212     Node newRecordsElement = XMLLib.newNode("RECORDS");
    213     while(null != currentDimensionIndexes) {
    214 
    215       // get the xpath for the current ibisq query result RECORD element -
    216       // dimension name/value (based on the indexes) to be queried.
    217       String queryResultRecordXPath = getQueryResultRecordXPath(
    218         dimensionNameValuesMap,
    219         dimensionName,
    220         currentDimensionIndexes
    221       );
    222 
    223       // test if that record exists.  If it doesn't then add it.  If it does
    224       // exist then get xpath the doc's record and detach it so it can be
    225       // added to the new set.
    226       int missingValueRecordCount = 0;
    227       try {
    228         XPathExpression xPathExpression = xpath.compile(queryResultRecordXPath);
    229         NodeList nodeList = (NodeList)xPathExpression.evaluate(w3cQueryModuleDocument, XPathConstants.NODESET);
    230         missingValueRecordCount = nodeList.getLength();
    231       }
    232       catch(Exception e) {
    233         missingValueRecordCount = 0;
    234         logger.error(".addMissingValues - xpath compile/evaluate cause: {}", e.getCause().getLocalizedMessage()); 
    235       }
    236       if(0 == missingValueRecordCount) {
    237         logger.debug(".addMissingQueryResultRecordElements - adding missing record for dimension: {}.", currentDimensionIndexes);
    238         XMLLib.addNode(newRecordsElement,
    239            createNewMissingValueRecordElement(
    240              queryModuleConfigurationNode,
    241              dimensionNameValuesMap,
    242              dimensionName,
    243              currentDimensionIndexes
    244            )
    245         );
    246       }
    247       else {
    248         logger.debug(".addMissingQueryResultRecordElements - found record for dimension: {}, xpath:{}.", currentDimensionIndexes, queryResultRecordXPath);
    249         Node nodeToAdd = XMLLib.getNode(queryModuleConfigurationNode, queryResultRecordXPath);
    250         if(null != nodeToAdd) {
    251           XMLLib.addNode(newRecordsElement, nodeToAdd.detach());
    252         }
    253         else {
    254           logger.error(".addMissingQueryResultRecordElements - found record but xpath returned NULL node.  xpath:{}.", queryResultRecordXPath);
    255         }
    256       }
    257 
    258       // Increment the indexes/counters and continue until done.
    259       currentDimensionIndexes = arrayIndexIncrementer.incrementCurrentIndex();
    260     }
    261 
    262     // finally, replace the existing RECORDS element with the newly created version. 
    263     Node ibisQResultContainer = XMLLib.getNode(queryModuleConfigurationNode, "/QUERY_MODULE/IBISQ_QUERY_RESULT");
    264     XMLLib.replaceNode(ibisQResultContainer, "RECORDS", newRecordsElement);
    265   } //-------------------------- End of Method ------------------------------
    266 
    267 
    268 
    269   /**
    270    * Builds a valid values map of all query result dimension name/values based
    271    * on missingValuesDimensionNameList and the actual query result record's dim
    272    * names and values.  The map entries built have a dimension name as its key
    273    * with the value of the map entry being a List of associated dimension values.
    274    *
    275    * This valid list of values is all query result dim values for those dims that
    276    * do not match one of the dims passed in in the list.  If in the list then
    277    * dim and its values are checked for filtering.  If not being filtered then
    278    * the entire dim's values are included.  If filtered then only the filtered
    279    * values are included.  This code also handles if the dim list name is a
    280    * proxy to the actual query result.
    281    * @param queryModuleDocument
    282    * @param missingValuesDimensionNameList optional list of dimension names to
    283    *   be processed.  If null then all dim names and values will be built.
    284    * @return map of all dimension name/values IF criteria is matched otherwise
    285    *   a null map is returned - which is a flag to not process.
    286    */
    287   protected Map<String, List<String>> getDimensionValuesMap (
    288       Node queryModuleDocument,
    289       List<String> missingValuesDimensionNameList
    290   ) {
    291 
    292 
    293 get the  QM/REQUEST/
    294         SELECTED_GROUP_BY
    295             CATEGORY_DIMENSION_NAME
    296             SERIES_DIMENSION_NAME
    297             OTHER_DIMENSION_NAME
    298         ACTUAL_GROUP_BY
    299             CATEGORY_DIMENSION_NAME
    300             SERIES_DIMENSION_NAME
    301             OTHER_DIMENSION_NAME
    302 
    303 if all then use the ACTUAL group by dim names to get the values
    304 
    305 else need to test the ADD MISSING DIM NAME
    306   check all SELECTED_GROUP_BY first. 
    307     if matches MISSING DIM NAME then get associated ACTUAL and use
    308   else check MISSING DIM NAME against ACTUAL
    309     if matches then use
    310 
    311 build the list of valid DIM VALUES by checking filtering.  If filtered then
    312 dim values to use are the filtered ones.  Else - use all dim values that
    313 do NOT have a NOT_SELECTABLE_FLAG. E.g. Only process missing dim values that 
    314 DO NOT have the NOT_SELE flag. 
     352    Node recordsContainerElement = XMLLib.getNode(queryModuleDocument, "/QUERY_MODULE/IBISQ_QUERY_RESULT/RECORDS");
     353    String[] dimensionNameKey = dimensionNameValuesMap.keySet().toArray(new String[3]);
     354    String[] dimensionValues  = new String[3];
     355    List<String> dimension1Values = dimensionNameValuesMap.get(dimensionNameKey[0]);
     356    List<String> dimension2Values = (1 < dimensionNameValuesMap.size()) ? dimensionNameValuesMap.get(dimensionNameKey[1]) : null;
     357    List<String> dimension3Values = (2 < dimensionNameValuesMap.size()) ? dimensionNameValuesMap.get(dimensionNameKey[2]) : null;
     358    List<Node> ancillaryValueNameList = (null != ancillaryValueNames) ? XMLLib.getNodes(ancillaryValueNames, "ANCILLARY_VALUE_NAME") : null;
    315359   
    316    
    317    
    318    
    319    
    320    
    321    
    322     // First, get the dimension names and associated values (map - contains
    323     // the dim name and a list of the associated dim values).  If the dim is
    324     // one of the "populate missing values" list then 1) if filtering on that
    325     // dim use the filtered values else 2) use ALL the dimension values from
    326     // the defined dimension list.  If dimension is not missing values then
    327     // get the unique, actual dim values from the query result.
    328     // NOTE: tried doing an xpath of ::preceeding-sibling but did NOT work so
    329     // went with brute force of looping all and keeping track of what was
    330     // processed.  XSLT 2.0 supports distinct-values but this is xpath so NA.
    331     // NOTE 2: Using a linked hash map to retain ordering.
    332 
    333     // global flag that if false means that nothing matched so leave result as is - return a null list.
    334     boolean processForMissingValues = false;
    335 
    336     List<Node> queryResultDimensionNameNodeList = XMLLib.getNodes(queryModuleDocument, "/QUERY_MODULE/IBISQ_QUERY_RESULT//DIMENSION/NAME");
    337     List<String> processedDimensionNames = new ArrayList<String>();
    338     Map<String, List<String>> dimensionNameValuesMap = new LinkedHashMap<String, List<String>>();
    339     for(int i=0; i<queryResultDimensionNameNodeList.size(); i++) {
    340       List<Node> dimensionValuesNodeList;
    341       List<String> dimensionValuesList = new ArrayList<String>();
    342       String dimensionName = queryResultDimensionNameNodeList.get(i).getText().trim();
    343       if(!processedDimensionNames.contains(dimensionName)) {
    344 
    345         // Process for missing values if:
    346         // 1) the QM config has the ALL_FLAG set e.g. null missingValuesDimensionNameList
    347         // 2) if the result has a dimension name that matches a missingValuesDimensionNameList.
    348         // 3) if the result has a dim name that matches a QM dim name and that QM dim has
    349         //      a proxy name that is contained in the missingValuesDimensionNameList.
    350         boolean addMissingValues = (null == missingValuesDimensionNameList);
    351         if(!addMissingValues) addMissingValues = missingValuesDimensionNameList.contains(dimensionName);
    352         if(!addMissingValues){
    353           String proxyDimensionName = XMLLib.getText(
    354             queryModuleDocument,
    355             "/QUERY_MODULE/DIMENSIONS/DIMENSION[NAME='" + dimensionName + "']/PROXY_DIMENSION_NAME",
    356             ""
    357           );
    358           addMissingValues = missingValuesDimensionNameList.contains(proxyDimensionName);
    359         }
    360 
    361         if(addMissingValues){
    362           logger.debug(".getMissingValueDimensionValuesMap - processForMissingValues, IBIS-Q XML: " + XMLLib.getNode(queryModuleDocument, "/QUERY_MODULE/IBISQ_QUERY_RESULT").asXML());
    363           processForMissingValues = true;   // set the global flag...
    364 
    365           // first try and select "selected" dimension values.  If there are
    366           // not any for the given dimension, then select ALL dimension values
    367           // from the core DIMENSION definition.  IF there are any result
    368           // values that are NOT_ASSOCITED those need to be included.
    369           dimensionValuesNodeList = XMLLib.getNodes(
    370               queryModuleDocument,
    371               "/QUERY_MODULE/CRITERIA/SECTIONS/SECTION/SELECTED_DIMENSIONS/SELECTED_DIMENSION[NAME = '" + dimensionName + "']/VALUES/VALUE"
    372           );
    373           if(dimensionValuesNodeList.size() < 1) {
    374             dimensionValuesNodeList = XMLLib.getNodes(
    375                 queryModuleDocument,
    376                 "/QUERY_MODULE/DIMENSIONS/DIMENSION[NAME = '" + dimensionName + "']/VALUES/VALUE[not(NOT_ASSOCIATED_FLAG)]"
    377             );
     360    int k = (null != dimension3Values) ? dimension3Values.size() : 1;
     361    while(0 < k) {
     362      k--;
     363      dimensionValues[2] = (null != dimension3Values) ? dimension3Values.get(k) : null;
     364
     365      int j = (null != dimension2Values) ? dimension2Values.size() : 1;
     366      while(0 < j) {
     367        j--;
     368        dimensionValues[1] = (null != dimension2Values) ? dimension2Values.get(j) : null;
     369
     370        int i = dimension1Values.size();
     371        while(0 < i) {
     372          i--;
     373          dimensionValues[0] = dimension1Values.get(i);
     374
     375          // get the xpath for the current ibisq query result RECORD element -
     376          // dimension name/value (based on the indexes) to be queried.
     377          String queryResultRecordXPath = getQueryResultRecordXPath(dimensionNameKey, dimensionValues);
     378
     379          // test if that record exists.  If it doesn't then add it.  If it does
     380          // exist then get xpath the doc's record and detach it so it can be
     381          // added to the new set.
     382          int missingValueRecordCount = 0;
     383          try {
     384            XPathExpression xPathExpression = xpath.compile(queryResultRecordXPath);
     385            NodeList nodeList = (NodeList)xPathExpression.evaluate(w3cQueryModuleDocument, XPathConstants.NODESET);
     386            missingValueRecordCount = nodeList.getLength();
    378387          }
    379 
    380           for(int j=0; j<dimensionValuesNodeList.size(); j++) {
    381             String dimensionValue = dimensionValuesNodeList.get(j).getText().trim();
    382             dimensionValuesList.add(dimensionValue);
     388          catch(Exception e) {
     389            missingValueRecordCount = 0;
     390            logger.error(".addMissingResultRecordElements - xpath compile/evaluate cause: {}", e.getCause().getLocalizedMessage()); 
    383391          }
    384 
    385           // need to process any other values returned - like NOT_ASSOCIATEDs
    386           // or any other value the query returned and is not in the dim list.
    387           // So, grab all actual values and if the list doesn't contain then
    388           // add it to the list...
    389           dimensionValuesNodeList = XMLLib.getNodes(queryModuleDocument,
    390               "/QUERY_MODULE/IBISQ_QUERY_RESULT//DIMENSION" +
    391               "[NAME = '" + dimensionName + "']/VALUE"
    392           );
    393           for(int j=0; j<dimensionValuesNodeList.size(); j++) {
    394             String dimensionValue = dimensionValuesNodeList.get(j).getText().trim();
    395             if(!dimensionValuesList.contains(dimensionValue)) {
    396               dimensionValuesList.add(dimensionValue);
    397             }
     392          if(0 == missingValueRecordCount) {
     393            logger.debug(".addMissingResultRecordElements - adding missing record for dimension names: {}, values: {}.", dimensionNameKey, dimensionValues);
     394            XMLLib.addNode(recordsContainerElement,
     395              createNewMissingValueRecordElement(
     396                dimensionNameKey, dimensionValues,
     397                measureName, missingMeasureValue,
     398                ancillaryValueNameList
     399              )
     400            );
    398401          }
    399402        }
    400 
    401         // Else this dim name (and its values) are not part of the missing name
    402         // mechanism - add all the values...
    403         else {
    404           dimensionValuesNodeList = XMLLib.getNodes(queryModuleDocument,
    405               "/QUERY_MODULE/IBISQ_QUERY_RESULT//DIMENSION" +
    406               "[NAME = '" + dimensionName + "']/VALUE"
    407           );
    408           for(int j=0; j<dimensionValuesNodeList.size(); j++) {
    409             String dimensionValue = dimensionValuesNodeList.get(j).getText().trim();
    410             if(!dimensionValuesList.contains(dimensionValue)) {
    411               dimensionValuesList.add(dimensionValue);
    412             }
    413           }
    414         }
    415 
    416         // now add the dim values to the dim map and add the processed dim name
    417         // to the list so that it won't be processed twice.
    418         dimensionNameValuesMap.put(dimensionName, dimensionValuesList);
    419         processedDimensionNames.add(dimensionName);
    420       }
    421     }
    422 
    423     // if missing values exist to be processed then return the map.  Else null.
    424     if(processForMissingValues)
    425       return(dimensionNameValuesMap);
    426     else
    427       return(null);
     403      }
     404    }
     405
    428406  } //-------------------------- End of Method ------------------------------
    429407
    … …  
    433411   * Creates a stubbed out IBIS_QUERY_RESULT//RECORD element based on the
    434412   * dimensions and CONFIGURATION//MEASURE/MISSING_VALUE value.
    435    * @param queryModuleConfigurationNode
    436    * @param dimensionNameValuesMap
    437    * @param dimensionName
    438    * @param currentDimensionIndexes
    439413   * @return complete IBIS_QUERY_RESULT//RECORD element.
    440414   */
    441415  protected Node createNewMissingValueRecordElement(
    442     Node queryModuleConfigurationNode,
    443     Map<String, List<String>> dimensionNameValuesMap,
    444     String[] dimensionName,
    445     int[] currentDimensionIndexes
     416    String[] dimensionName, String[] dimensionValue,
     417    String measureName, String missingValue,
     418    List<Node> ancillaryValueNameList
    446419  ) {
     420    Node recordElement = XMLLib.newNode("RECORD");
     421
    447422    // create the DIMENSIONS container and add all dims according to the index.
    448423    Node dimensionsContainerElement = XMLLib.newNode("DIMENSIONS");
    449     for(int i=0; i<dimensionName.length; i++) {
    450       Node dimensionElement = XMLLib.newNode("DIMENSION");
    451       XMLLib.addNode(dimensionElement, XMLLib.newNode("NAME", dimensionName[i]));
    452 
    453       List<String> dimensionValues = dimensionNameValuesMap.get(dimensionName[i]);
    454       String value = (String)dimensionValues.get(currentDimensionIndexes[i]);
    455       XMLLib.addNode(dimensionElement, XMLLib.newNode("VALUE", value));
    456       XMLLib.addNode(dimensionsContainerElement, dimensionElement);
    457     }
    458 
    459     // create the MEASURES container and add all MEASURE elements according to
    460     // the CONFIGURATION passed in.
    461     Node measuresContainerElement = XMLLib.newNode("MEASURES");
    462     List<Node> measuresList = XMLLib.getNodes(queryModuleConfigurationNode, ".//MEASURE");
    463     for(int i=0; i < measuresList.size(); i++) {
    464       Node nameElement         = XMLLib.getNode(measuresList.get(i), "NAME");
    465       Node missingValueElement = XMLLib.getNode(measuresList.get(i), "MISSING_QUERY_RESULT_RECORD_VALUE");
    466       String missingValue = this.defaultMissingValue;
    467       if(null != missingValueElement) missingValue = XMLLib.getText(missingValueElement).trim();
    468 
    469       Node measureElement = XMLLib.newNode("MEASURE");
    470       XMLLib.addNode(measureElement, nameElement);
    471       XMLLib.addNode(measureElement, XMLLib.newNode("VALUE", missingValue));
    472       XMLLib.addNode(measuresContainerElement, measureElement);
    473     }
    474 
    475     // finally add the DIMENSIONS and MEASURES to a new RECORD element and return.
    476     Node recordElement = XMLLib.newNode("RECORD");
     424    for(int i=0; i< dimensionName.length; i++) {
     425      addNewMissingValueDimensionElement(dimensionsContainerElement, dimensionName[i], dimensionValue[i]);
     426    }
    477427    XMLLib.addNode(recordElement, dimensionsContainerElement);
    478     XMLLib.addNode(recordElement, measuresContainerElement);
     428
     429    // create the MEASURE element.
     430    Node measureElement = XMLLib.newNode("MEASURE");
     431    XMLLib.addNode(measureElement, XMLLib.newNode("NAME", measureName));
     432    XMLLib.addNode(measureElement, XMLLib.newNode("VALUE", missingValue));
     433    XMLLib.addNode(recordElement, measureElement);
     434
     435    // create the ANCILLARY container and add all the supplied AVs.
     436    if(null != ancillaryValueNameList) {
     437      Node ancillaryValueContainerElement = XMLLib.newNode("ANCILLARY_VALUES");
     438      for(Node ancillaryValueName : ancillaryValueNameList) {
     439        Node ancillaryValueElement = XMLLib.newNode("ANCILLARY_VALUE");
     440        XMLLib.addNode(ancillaryValueElement, XMLLib.newNode("NAME",  XMLLib.getText(ancillaryValueName)));
     441        XMLLib.addNode(ancillaryValueElement, XMLLib.newNode("VALUE", missingValue));
     442        XMLLib.addNode(ancillaryValueContainerElement, ancillaryValueElement);
     443      }
     444      XMLLib.addNode(recordElement, ancillaryValueContainerElement);
     445    }
    479446
    480447    logger.debug(".createNewMissingValueRecordElement - XML Created: " + recordElement.asXML());
    … …  
    482449  } //-------------------------- End of Method ------------------------------
    483450
     451
     452  protected void addNewMissingValueDimensionElement(
     453    Node dimensionsContainerElement,
     454    String dimensionName,
     455    String dimensionValue
     456  ) {
     457    if(StrLib.isSomething(dimensionName)) {
     458      Node dimensionElement = XMLLib.newNode("DIMENSION");
     459      XMLLib.addNode(dimensionElement, XMLLib.newNode("NAME", dimensionName));
     460      XMLLib.addNode(dimensionElement, XMLLib.newNode("VALUE", dimensionValue));
     461      XMLLib.addNode(dimensionsContainerElement, dimensionElement);
     462    }
     463  } //-------------------------- End of Method ------------------------------
    484464
    485465
    … …  
    489469   * @return xpath for the RECORD to be retrieved.
    490470   */
    491   protected String getQueryResultRecordXPath(
    492       Map<String, List<String>> dimensionNameValuesMap,
    493       String[] dimensionName,
    494       int[] dimensionIndex
    495   ) {
     471  protected String getQueryResultRecordXPath(String[] dimensionName, String[] dimensionValue) {
    496472    StringBuffer xpath = new StringBuffer();
     473    boolean addAnAnd = false;
    497474    xpath.append("/QUERY_MODULE/IBISQ_QUERY_RESULT/RECORDS/RECORD[DIMENSIONS[");
    498     for(int i=0; i<dimensionName.length;) {
    499       xpath.append("(DIMENSION[NAME='");
    500       xpath.append(dimensionName[i]);
    501       xpath.append("' and VALUE/text()='");
    502       List<String> dimensionValues = dimensionNameValuesMap.get(dimensionName[i]);
    503       xpath.append(dimensionValues.get(dimensionIndex[i]));
    504       xpath.append("'])");
    505       i++;
    506       if(i < dimensionName.length) xpath.append(" and ");
     475    for(int i=0; i<dimensionName.length; i++) {
     476      if((null != dimensionName[i]) && (null != dimensionValue[i])) {
     477        if(addAnAnd) xpath.append(" and ");
     478        xpath.append("(DIMENSION[NAME='");
     479        xpath.append(dimensionName[i]);
     480        xpath.append("' and VALUE/text()='");
     481        xpath.append(dimensionValue[i]);
     482        xpath.append("'])");
     483        addAnAnd = true;
     484      }
    507485    }
    508486    xpath.append("]]");
    … …  
    512490
    513491} //============================ END OF CLASS =================================
    514 
Note: See TracChangeset for help on using the changeset viewer.