Neptune Analytics - Logger

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aws/neptune-for-graphql",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "CLI utility to create and maintain a GraphQL API for Amazon Neptune",
   "keywords": [
     "Amazon Neptune",

src/CDKPipelineApp.js

@@ -16,6 +16,7 @@ import { readFile, writeFile } from 'fs/promises';
 import fs from 'fs';
 import archiver from 'archiver';
 import ora from 'ora';
+import { loggerLog } from "./logger.js";
 
 let NAME = '';
 let REGION = '';
@@ -32,7 +33,7 @@ let APPSYNC_ATTACH_QUERY = [];
 let APPSYNC_ATTACH_MUTATION = [];
 let SCHEMA_MODEL = null;
 let thisOutputFolderPath = './output';
-
+let msg = '';
 
 function yellow(text) {
     return '\x1b[33m' + text + '\x1b[0m';
@@ -68,7 +69,9 @@ async function createDeploymentFile(folderPath, zipFilePath) {
         archive.file('./output/output.resolver.graphql.js', { name: 'output.resolver.graphql.js' })
         await archive.finalize();
     } catch (err) {
-        console.error('Creating deployment zip file: ' + err); 
+        msg = 'Creating deployment zip file: ' + err;
+        console.error(msg); 
+        loggerLog(msg);
     }
 }
 
@@ -125,6 +128,8 @@ async function createAWSpipelineCDK (pipelineName, neptuneDBName, neptuneDBregio
         NEPTUNE_IAM_POLICY_RESOURCE = neptuneClusterInfo.iamPolicyResource;      
 
     } catch (error) {
+        msg = 'Error getting Neptune Cluster Info: ' + JSON.stringify(error);
+        loggerLog(msg);
         if (!quiet) spinner.fail("Error getting Neptune Cluster Info.");
         if (!isNeptuneIAMAuth) {
             spinner.clear();

src/NeptuneSchema.js

@@ -14,15 +14,17 @@ import axios from "axios";
 import { aws4Interceptor } from "aws4-axios";
 import { fromNodeProviderChain  } from "@aws-sdk/credential-providers";
 import { NeptunedataClient, ExecuteOpenCypherQueryCommand } from "@aws-sdk/client-neptunedata";
+import { loggerLog } from "./logger.js";
 
 let HOST = '';
 let PORT = 8182;
 let REGION = ''
 let SAMPLE = 5000;
-let VERBOSE = false; 
+let VERBOSE = false;
+let NEPTUNE_TYPE = 'neptune-db'; 
 let language = 'openCypher';
 let useSDK = false;
-
+let msg = '';
 
 async function getAWSCredentials() {
     const credentialProvider = fromNodeProviderChain();
@@ -31,7 +33,7 @@ async function getAWSCredentials() {
     const interceptor = aws4Interceptor({
         options: {
             region: REGION,
-            service: "neptune-db",
+            service: NEPTUNE_TYPE,
         },
         credentials: cred
     });
@@ -55,6 +57,7 @@ function consoleOut(text) {
     if (VERBOSE) {
         console.log(text);
     }
+    loggerLog(text);
 }
 
 
@@ -67,11 +70,18 @@ async function queryNeptune(q) {
         const response = await axios.post(`https://${HOST}:${PORT}/${language}`, `query=${encodeURIComponent(q)}`);
         return response.data;    
         } catch (error) {
-            console.error("Http query request failed: ", error.message);
-            consoleOut("Trying with the AWS SDK");            
-            const response = await queryNeptuneSDK(q);
-            useSDK = true;
-            return response;             
+            msg = `Http query request failed: ${error.message}`;
+            consoleOut.error(msg);
+            loggerLog(msg + ': ' + JSON.stringify(error));
+
+            if (NEPTUNE_TYPE == 'neptune-db') {
+                consoleOut("Trying with the AWS SDK");            
+                const response = await queryNeptuneSDK(q);
+                useSDK = true;
+                return response; 
+            }
+
+            throw new Error('AWS SDK for Neptune Analytics is not available, yet.');
         }
     } 
 }
@@ -91,7 +101,9 @@ async function queryNeptuneSDK(q) {
         return response;
 
     } catch (error) {        
-        console.error("SDK query request failed: ", error.message);
+        msg = `SDK query request failed: ${error.message}`;
+        consoleOut.error(msg);
+        loggerLog(msg + ': ' + JSON.stringify(error));        
         process.exit(1);
     }
 }
@@ -100,15 +112,18 @@ async function queryNeptuneSDK(q) {
 async function getNodesNames() {
     let query = `MATCH (a) RETURN labels(a), count(a)`;
     let response = await queryNeptune(query);    
+    loggerLog('Getting nodes names');    
 
     try {
         response.results.forEach(result => {
-            schema.nodeStructures.push({ label: result['labels(a)'][0], properties: []});
+            schema.nodeStructures.push({ label: result['labels(a)'][0], properties: []});            
             consoleOut('  Found Node: ' + yellow(result['labels(a)'][0]));
         });        
     }
     catch (e)  {
-        consoleOut("  No nodes found");
+        msg = "  No nodes found";
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
         return;
     }
 }
@@ -117,6 +132,7 @@ async function getNodesNames() {
 async function getEdgesNames() {
     let query = `MATCH ()-[e]->() RETURN type(e), count(e)`;
     let response = await queryNeptune(query);
+    loggerLog('Getting edges names');
 
     try {
         response.results.forEach(result => {
@@ -125,35 +141,31 @@ async function getEdgesNames() {
         });
     }
     catch (e)  {
-        consoleOut("  No edges found");
+        msg = "  No edges found";
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
         return;
     }
 
 }
 
 
-async function checkEdgeDirection(direction) {
-    let query = `MATCH (from:${direction.from})-[r:${direction.edge.label}]->(to:${direction.to}) RETURN r as edge LIMIT 1`;
+async function findFromAndToLabels(edgeStructure) {
+    let query = `MATCH (from)-[r:${edgeStructure.label}]->(to) RETURN DISTINCT labels(from) as fromLabel, labels(to) as toLabel`;
     let response = await queryNeptune(query);
-    let result = response.results[0];
-    if (result !== undefined) {                    
-        direction.edge.directions.push({from:direction.from, to:direction.to});
-        consoleOut('  Found edge: ' + yellow(direction.edge.label) + '  direction: ' + yellow(direction.from) + ' -> ' + yellow(direction.to));
+    for (let result of response.results) {
+        for (let fromLabel of result.fromLabel) {
+            for (let toLabel of result.toLabel) {
+                edgeStructure.directions.push({from:fromLabel, to:toLabel});
+                consoleOut('  Found edge: ' + yellow(edgeStructure.label) + '  direction: ' + yellow(fromLabel) + ' -> ' + yellow(toLabel));
+            }
+        }
     }
 }
 
 
 async function getEdgesDirections() {
-    let possibleDirections = [];
-    for (const edge of schema.edgeStructures) {        
-        for (const fromNode of schema.nodeStructures) {
-            for (const toNode of schema.nodeStructures) {
-                possibleDirections.push({edge:edge, from:fromNode.label, to:toNode.label});            
-            }
-        }
-    }
-
-    await Promise.all(possibleDirections.map(checkEdgeDirection))
+    await Promise.all(schema.edgeStructures.map(findFromAndToLabels))
 }
 
 
@@ -196,7 +208,8 @@ function addUpdateEdgeProperty(edgeName, name, value) {
 
 
 async function getEdgeProperties(edge) {
-    let query = `MATCH ()-[n:${edge.label}]->() RETURN properties(n) as properties LIMIT ${SAMPLE}`;        
+    let query = `MATCH ()-[n:${edge.label}]->() RETURN properties(n) as properties LIMIT ${SAMPLE}`;
+    loggerLog(`Getting properties for edge: ${query}`);        
     try {
         let response = await queryNeptune(query);            
         let result = response.results;
@@ -207,7 +220,9 @@ async function getEdgeProperties(edge) {
         });            
     }
     catch (e)  {
-        consoleOut("  No properties found for edge: " + edge.label);
+        msg = "  No properties found for edge: " + edge.label;
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
     }    
 }
 
@@ -220,7 +235,8 @@ async function getEdgesProperties() {
 
 
 async function getNodeProperties(node) {
-    let query = `MATCH (n:${node.label}) RETURN properties(n) as properties LIMIT ${SAMPLE}`;        
+    let query = `MATCH (n:${node.label}) RETURN properties(n) as properties LIMIT ${SAMPLE}`;
+    loggerLog(`Getting properties for node: ${query}`);
     try {
         let response = await queryNeptune(query);            
         let result = response.results;
@@ -231,7 +247,9 @@ async function getNodeProperties(node) {
         });            
     }
     catch (e)  {
-        consoleOut("  No properties found for node: " + node.label);
+        msg = "  No properties found for node: " + node.label;
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
     }    
 }
 
@@ -244,10 +262,12 @@ async function getNodesProperties() {
 
 
 async function checkEdgeDirectionCardinality(d) {
-    let queryFrom = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH to, count(from) as rels WHERE rels > 1 RETURN rels LIMIT 1`;     
+    let queryFrom = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH to, count(from) as rels WHERE rels > 1 RETURN rels LIMIT 1`;
+    loggerLog(`Checking edge direction cardinality: ${queryFrom}`);     
     let responseFrom = await queryNeptune(queryFrom);
     let resultFrom = responseFrom.results[0];
-    let queryTo = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH from, count(to) as rels WHERE rels > 1 RETURN rels LIMIT 1`;            
+    let queryTo = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH from, count(to) as rels WHERE rels > 1 RETURN rels LIMIT 1`;
+    loggerLog(`Checking edge direction cardinality: ${queryTo}`);
     let responseTo = await queryNeptune(queryTo);
     let resultTo = responseTo.results[0];
     let c = '';
@@ -283,11 +303,12 @@ async function getEdgesDirectionsCardinality() {
 }
 
 
-function setGetNeptuneSchemaParameters(host, port, region, verbose = false) {
+function setGetNeptuneSchemaParameters(host, port, region, verbose = false, neptuneType) {
     HOST = host;
     PORT = port;
     REGION = region;
     VERBOSE = verbose;
+    NEPTUNE_TYPE = neptuneType;
 }
 
 
@@ -307,6 +328,7 @@ async function getSchemaViaSummaryAPI() {
         return true;
 
     } catch (error) {
+        loggerLog(`Getting the schema via Neptune Summary API failed: ${JSON.stringify(error)}`);
         return false;
     }    
 }
@@ -318,8 +340,10 @@ async function getNeptuneSchema(quiet) {
 
     try {
         await getAWSCredentials();
-    } catch (error) {        
-        consoleOut("There are no AWS credetials configured. \nGetting the schema from an Amazon Neptune database with IAM authentication works only with AWS credentials.");
+    } catch (error) {
+        msg = "There are no AWS credetials configured. \nGetting the schema from an Amazon Neptune database with IAM authentication works only with AWS credentials.";        
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(error));
     }
 
     if (await getSchemaViaSummaryAPI()) {

src/graphdb.js

@@ -69,12 +69,16 @@ function graphDBInferenceSchema (graphbSchema, addMutations) {
         }
 
         r += '\t_id: ID! @id\n';
-
+
+        let properties = [];
         node.properties.forEach(property => {
+            properties.push(property.name);
+
             if (property.name == 'id')                
                 r+= `\tid: ID\n`;
             else
                 r+= `\t${property.name}: ${property.type}\n`;
+
         });
 
         let edgeTypes = [];
@@ -127,11 +131,15 @@ function graphDBInferenceSchema (graphbSchema, addMutations) {
         });
 
         // Add edge types        
-        edgeTypes.forEach(edgeType => {
+        edgeTypes.forEach(edgeType => {            
+            let collision = '';
+            if (properties.includes(edgeType))
+                collision = '_';
+
             if (changeCase) {
-                r += `\t${edgeType}:${toPascalCase(edgeType)}`
+                r += `\t${collision + edgeType}:${toPascalCase(edgeType)}`
             } else {
-                r += `\t${edgeType}:${edgeType}`
+                r += `\t${collision + edgeType}:${edgeType}`
             }
         });
 

src/lambdaZip.js

@@ -1,5 +1,8 @@
 import fs from 'fs';
 import archiver from 'archiver';
+import { loggerLog } from "./logger.js";
+
+let msg = '';
 
 async function createLambdaDeploymentPackage(templatePath, zipFilePath) {
     try {       
@@ -10,7 +13,9 @@ async function createLambdaDeploymentPackage(templatePath, zipFilePath) {
         archive.file('./output/output.resolver.graphql.js', { name: 'output.resolver.graphql.js' })
         await archive.finalize();
     } catch (error) {
-        console.error('Lambda deployment package creation failed. '+ error);
+        msg = 'Lambda deployment zip file: ' + JSON.stringify(error);
+        loggerLog(msg);
+        console.error('Lambda deployment package creation failed. '+ error.message);
     }
 }
 

src/logger.js

@@ -0,0 +1,22 @@
+import fs from 'fs';
+
+let LOG_FILE = './output/log.txt';
+
+function loggerInit(file) {
+    LOG_FILE = file;
+    fs.writeFileSync(LOG_FILE, '', (err) => {
+        return console.log(err);
+    });
+}
+
+function loggerLog(text) {
+    // remove yellow escape from text 
+    text = text.replaceAll(/\x1b\[33m/g, '');
+    text = text.replaceAll(/\x1b\[0m/g, '');
+
+    fs.appendFileSync(LOG_FILE, (new Date()).toISOString() + ' ' + text + '\n', (err) => {
+        return console.log(err);
+    })
+}
+
+export { loggerInit, loggerLog };