Add end to end tracing header and refactore resource header

OBSERVABILITY.md

@@ -80,7 +80,27 @@ const spanner = new Spanner({
   observabilityOptions: {
     tracerProvider: provider,
     enableExtendedTracing: true,
-  },
+  }
+}),
+```
+
+#### End to end tracing
+
+In addition to client-side tracing, you can opt in for end-to-end tracing. End-to-end tracing helps you understand and debug latency issues that are specific to Spanner. Refer [here](https://cloud.google.com/spanner/docs/tracing-overview) for more information.
+
+You can opt-in by either:
+
+* Setting the environment variable `SPANNER_ENABLE_END_TO_END_TRACING=true` before your application is started
+* In code, setting `enableEndToEndTracing: true` in your SpannerOptions before creating the Cloud Spanner client
+
+```javascript
+const spanner = new Spanner({
+  projectId: projectId,
+  observabilityOptions: {
+    tracerProvider: provider,
+    enableEndToEndTracing: true,
+  }
+}),
 ```
 
 #### OpenTelemetry gRPC instrumentation

observability-test/spanner.ts

@@ -26,6 +26,7 @@ import * as mockInstanceAdmin from '../test/mockserver/mockinstanceadmin';
 import * as mockDatabaseAdmin from '../test/mockserver/mockdatabaseadmin';
 import * as sinon from 'sinon';
 import {Row} from '../src/partial-result-stream';
+import {END_TO_END_TRACING_HEADER} from '../src/common';
 const {
   AlwaysOnSampler,
   NodeTracerProvider,
@@ -1930,3 +1931,59 @@ describe('Traces for ExecuteStream broken stream retries', () => {
     );
   });
 });
+
+describe('End to end tracing headers', () => {
+  let server: grpc.Server;
+  let spanner: Spanner;
+  let spannerMock: mock.MockSpanner;
+  let observabilityOptions: typeof ObservabilityOptions;
+
+  beforeEach(async () => {
+    observabilityOptions = {
+      enableEndToEndTracing: true,
+    };
+
+    const setupResult = await setup(observabilityOptions);
+    spanner = setupResult.spanner;
+    server = setupResult.server;
+    spannerMock = setupResult.spannerMock;
+  });
+
+  afterEach(async () => {
+    spannerMock.resetRequests();
+    spanner.close();
+    server.tryShutdown(() => {});
+  });
+
+  it('run', done => {
+    const instance = spanner.instance('instance');
+    const database = instance.database('database');
+    database.getTransaction((err, tx) => {
+      assert.ifError(err);
+
+      tx!.run('SELECT 1', async () => {
+        tx!.end();
+        let metadataCountWithE2EHeader = 0;
+        let metadataCountWithTraceParent = 0;
+        spannerMock.getMetadata().forEach(metadata => {
+          if (metadata.get(END_TO_END_TRACING_HEADER)[0] !== undefined) {
+            metadataCountWithE2EHeader++;
+            assert.strictEqual(
+              metadata.get(END_TO_END_TRACING_HEADER)[0],
+              'true'
+            );
+          }
+          if (metadata.get('traceparent')[0] !== undefined) {
+            metadataCountWithTraceParent++;
+          }
+        });
+
+        // Batch Create Session request and Select 1 request.
+        assert.strictEqual(spannerMock.getRequests().length, 2);
+        assert.strictEqual(metadataCountWithE2EHeader, 2);
+        assert.strictEqual(metadataCountWithTraceParent, 2);
+        done();
+      });
+    });
+  });
+});

src/common.ts

@@ -84,7 +84,7 @@ export const LEADER_AWARE_ROUTING_HEADER = 'x-goog-spanner-route-to-leader';
 /*
  * END TO END TRACING  header.
  */
-const END_TO_END_TRACING_HEADER = 'x-goog-spanner-end-to-end-tracing';
+export const END_TO_END_TRACING_HEADER = 'x-goog-spanner-end-to-end-tracing';
 
 /**
  * Add Leader aware routing header to existing header list.
@@ -104,7 +104,10 @@ export function getCommonHeaders(
 ) {
   const headers: {[k: string]: string} = {};
 
-  if (process.env.SPANNER_ENABLE_EXTENDED_TRACING === 'true' || enableTracing) {
+  if (
+    process.env.SPANNER_ENABLE_END_TO_END_TRACING === 'true' ||
+    enableTracing
+  ) {
     headers[END_TO_END_TRACING_HEADER] = 'true';
   }