Sharded timestamps

Stay organized with collections Save and categorize content based on your preferences.

asyncfunctioninsertData(){constinstruments=[{symbol:'AAA',price:{currency:'USD',micros:34790000},exchange:'EXCHG1',instrumentType:'commonstock',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.010Z'))},{symbol:'BBB',price:{currency:'JPY',micros:64272000000},exchange:'EXCHG2',instrumentType:'commonstock',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.101Z'))},{symbol:'Index1 ETF',price:{currency:'USD',micros:473000000},exchange:'EXCHG1',instrumentType:'etf',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.001Z'))}];constbatch=fs.batch();for(constinstofinstruments){constref=fs.collection('instruments').doc();batch.set(ref,inst);}awaitbatch.commit();}
nonShardedTimestamps.js

functioncreateQuery(fieldName,fieldOperator,fieldValue,limit=5){returnfs.collection('instruments').where(fieldName,fieldOperator,fieldValue).orderBy('timestamp','desc').limit(limit).get();}functionqueryCommonStock(){returncreateQuery('instrumentType','==','commonstock');}functionqueryExchange1Instruments(){returncreateQuery('exchange','==','EXCHG1');}functionqueryUSDInstruments(){returncreateQuery('price.currency','==','USD');}
nonShardedTimestamps.js

insertData().then(()=>{constcommonStock=queryCommonStock().then((docs)=>{console.log('--- queryCommonStock: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});constexchange1Instruments=queryExchange1Instruments().then((docs)=>{console.log('--- queryExchange1Instruments: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});constusdInstruments=queryUSDInstruments().then((docs)=>{console.log('--- queryUSDInstruments: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});returnPromise.all([commonStock,exchange1Instruments,usdInstruments]);});
nonShardedTimestamps.js

// Define our 'K' shard valuesconstshards=['x','y','z'];// Define a function to help 'chunk' our shards for use in queries.// When using the 'in' query filter there is a max number of values that can be// included in the value. If our number of shards is higher than that limit// break down the shards into the fewest possible number of chunks.functionshardChunks(){constchunks=[];letstart=0;while(start < shards.length){constelements=Math.min(MAX_IN_VALUES,shards.length-start);constend=start+elements;chunks.push(shards.slice(start,end));start=end;}returnchunks;}// Add a convenience function to select a random shardfunctionrandomShard(){returnshards[Math.floor(Math.random()*Math.floor(shards.length))];}
shardedTimestamps.js

asyncfunctioninsertData(){constinstruments=[{shard:randomShard(),// add the new shard field to the documentsymbol:'AAA',price:{currency:'USD',micros:34790000},exchange:'EXCHG1',instrumentType:'commonstock',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.010Z'))},{shard:randomShard(),// add the new shard field to the documentsymbol:'BBB',price:{currency:'JPY',micros:64272000000},exchange:'EXCHG2',instrumentType:'commonstock',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.101Z'))},{shard:randomShard(),// add the new shard field to the documentsymbol:'Index1 ETF',price:{currency:'USD',micros:473000000},exchange:'EXCHG1',instrumentType:'etf',timestamp:Timestamp.fromMillis(Date.parse('2019-01-01T13:45:23.001Z'))}];constbatch=fs.batch();for(constinstofinstruments){constref=fs.collection('instruments').doc();batch.set(ref,inst);}awaitbatch.commit();}
shardedTimestamps.js

functioncreateQuery(fieldName,fieldOperator,fieldValue,limit=5){// For each shard value, map it to a new query which adds an additional// where clause specifying the shard value.returnPromise.all(shardChunks().map(shardChunk=>{returnfs.collection('instruments').where('shard','in',shardChunk)// new shard condition.where(fieldName,fieldOperator,fieldValue).orderBy('timestamp','desc').limit(limit).get();}))// Now that we have a promise of multiple possible query results, we need// to merge the results from all of the queries into a single result set..then((snapshots)=>{// Create a new container for 'all' resultsconstdocs=[];snapshots.forEach((querySnapshot)=>{querySnapshot.forEach((doc)=>{// append each document to the new all containerdocs.push(doc);});});if(snapshots.length===1){// if only a single query was returned skip manual sorting as it is// taken care of by the backend.returndocs;}else{// When multiple query results are returned we need to sort the// results after they have been concatenated.// // since we're wanting the `limit` newest values, sort the array// descending and take the first `limit` values. By returning negated// values we can easily get a descending value.docs.sort((a,b)=>{constaT=a.data().timestamp;constbT=b.data().timestamp;constsecondsDiff=aT.seconds-bT.seconds;if(secondsDiff===0){return-(aT.nanoseconds-bT.nanoseconds);}else{return-secondsDiff;}});returndocs.slice(0,limit);}});}functionqueryCommonStock(){returncreateQuery('instrumentType','==','commonstock');}functionqueryExchange1Instruments(){returncreateQuery('exchange','==','EXCHG1');}functionqueryUSDInstruments(){returncreateQuery('price.currency','==','USD');}
shardedTimestamps.js

insertData().then(()=>{constcommonStock=queryCommonStock().then((docs)=>{console.log('--- queryCommonStock: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});constexchange1Instruments=queryExchange1Instruments().then((docs)=>{console.log('--- queryExchange1Instruments: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});constusdInstruments=queryUSDInstruments().then((docs)=>{console.log('--- queryUSDInstruments: ');docs.forEach((doc)=>{console.log(`doc = ${util.inspect(doc.data(),{depth:4})}`);});});returnPromise.all([commonStock,exchange1Instruments,usdInstruments]);});
shardedTimestamps.js