Update build files to latest versions

build/datatypes/dataset.d.ts

@@ -139,10 +139,12 @@ export declare class Dataset implements Instance<DatasetValue, DatasetJS> {
     splitFn(splitFns: Record<string, ComputeFn>, datasetName: string): Dataset;
     getReadyExternals(limit?: number): DatasetExternalAlterations;
     applyReadyExternals(alterations: DatasetExternalAlterations): Dataset;
+    sameKeys(other: Dataset): boolean;
+    getKeyValueForDatum(datum: Datum): string;
     getKeyLookup(): Record<string, Datum>;
     join(other: Dataset): Dataset;
     leftJoin(other: Dataset): Dataset;
-    fullJoin(other: Dataset, compare: (v1: any, v2: any) => number): Dataset;
+    fullJoin(other: Dataset): Dataset;
     findDatumByAttribute(attribute: string, value: any): Datum | undefined;
     getColumns(options?: FlattenOptions): AttributeInfo[];
     private _flattenHelper;

build/datatypes/dataset.js

@@ -3,6 +3,7 @@ import * as hasOwnProp from 'has-own-prop';
 import { generalEqual, NamedArray, SimpleArray } from 'immutable-class';
 import { Expression, ExternalExpression, LiteralExpression } from '../expressions/index';
 import { External, TotalContainer } from '../external/baseExternal';
+import { deduplicateSort } from '../helper';
 import { AttributeInfo } from './attributeInfo';
 import { datumHasExternal, valueFromJS, valueToJS } from './common';
 import { NumberRange } from './numberRange';
@@ -803,33 +804,44 @@ var Dataset = (function () {
         value.data = data;
         return new Dataset(value);
     };
+    Dataset.prototype.sameKeys = function (other) {
+        return this.keys.join('|') === other.keys.join('|');
+    };
+    Dataset.prototype.getKeyValueForDatum = function (datum) {
+        var keys = this.keys;
+        if (!keys)
+            throw new Error('join lhs must have a key (be a product of a split)');
+        return this.keys.map(function (k) {
+            var v = datum[k];
+            if (v && v.start)
+                v = v.start;
+            if (v && v.toISOString)
+                v = v.toISOString();
+            return v;
+        }).join('|');
+    };
     Dataset.prototype.getKeyLookup = function () {
         var _a = this, data = _a.data, keys = _a.keys;
-        var thisKey = keys[0];
-        if (!thisKey)
-            throw new Error('join lhs must have a key (be a product of a split)');
         var mapping = Object.create(null);
         for (var i = 0; i < data.length; i++) {
             var datum = data[i];
-            mapping[String(datum[thisKey])] = datum;
+            mapping[this.getKeyValueForDatum(datum)] = datum;
         }
         return mapping;
     };
     Dataset.prototype.join = function (other) {
         return this.leftJoin(other);
     };
     Dataset.prototype.leftJoin = function (other) {
+        var _this = this;
         if (!other || !other.data.length)
             return this;
         var _a = this, data = _a.data, keys = _a.keys, attributes = _a.attributes;
         if (!data.length)
             return this;
-        var thisKey = keys[0];
-        if (!thisKey)
-            throw new Error('join lhs must have a key (be a product of a split)');
         var otherLookup = other.getKeyLookup();
         var newData = data.map(function (datum) {
-            var otherDatum = otherLookup[String(datum[thisKey])];
+            var otherDatum = otherLookup[_this.getKeyValueForDatum(datum)];
             if (!otherDatum)
                 return datum;
             return joinDatums(datum, otherDatum);
@@ -840,51 +852,32 @@ var Dataset = (function () {
             data: newData
         });
     };
-    Dataset.prototype.fullJoin = function (other, compare) {
+    Dataset.prototype.fullJoin = function (other) {
         if (!other || !other.data.length)
             return this;
         var _a = this, data = _a.data, keys = _a.keys, attributes = _a.attributes;
         if (!data.length)
             return other;
-        var thisKey = keys[0];
-        if (!thisKey)
-            throw new Error('join lhs must have a key (be a product of a split)');
-        if (thisKey !== other.keys[0])
+        if (!this.sameKeys(other)) {
             throw new Error('this and other keys must match');
-        var otherData = other.data;
-        var dataLength = data.length;
-        var otherDataLength = otherData.length;
-        var newData = [];
-        var i = 0;
-        var j = 0;
-        while (i < dataLength || j < otherDataLength) {
-            if (i < dataLength && j < otherDataLength) {
-                var nextDatum = data[i];
-                var nextOtherDatum = otherData[j];
-                var cmp = compare(nextDatum[thisKey], nextOtherDatum[thisKey]);
-                if (cmp < 0) {
-                    newData.push(nextDatum);
-                    i++;
-                }
-                else if (cmp > 0) {
-                    newData.push(nextOtherDatum);
-                    j++;
+        }
+        var myDatumLookup = this.getKeyLookup();
+        var otherDatumLookup = other.getKeyLookup();
+        var newData = deduplicateSort(Object.keys(myDatumLookup).concat(Object.keys(otherDatumLookup))).map(function (key) {
+            var myDatum = myDatumLookup[key];
+            var otherDatum = otherDatumLookup[key];
+            if (myDatum) {
+                if (otherDatum) {
+                    return joinDatums(myDatum, otherDatum);
                 }
                 else {
-                    newData.push(joinDatums(nextDatum, nextOtherDatum));
-                    i++;
-                    j++;
+                    return myDatum;
                 }
             }
-            else if (i === dataLength) {
-                newData.push(otherData[j]);
-                j++;
-            }
             else {
-                newData.push(data[i]);
-                i++;
+                return otherDatum;
             }
-        }
+        });
         return new Dataset({
             keys: keys,
             attributes: AttributeInfo.override(attributes, other.attributes),

build/dialect/baseDialect.d.ts

@@ -10,6 +10,7 @@ export declare abstract class SQLDialect {
     maybeNamespacedName(name: string): string;
     escapeLiteral(name: string): string;
     booleanToSQL(bool: boolean): string;
+    floatDivision(numerator: string, denominator: string): string;
     numberOrTimeToSQL(x: number | Date): string;
     numberToSQL(num: number): string;
     dateToSQLDateString(date: Date): string;
@@ -28,7 +29,8 @@ export declare abstract class SQLDialect {
     abstract timeFloorExpression(operand: string, duration: Duration, timezone: Timezone): string;
     abstract timeBucketExpression(operand: string, duration: Duration, timezone: Timezone): string;
     abstract timePartExpression(operand: string, part: string, timezone: Timezone): string;
-    abstract timeShiftExpression(operand: string, duration: Duration, timezone: Timezone): string;
+    abstract timeShiftExpression(operand: string, duration: Duration, step: int, timezone: Timezone): string;
     abstract extractExpression(operand: string, regexp: string): string;
     abstract indexOfExpression(str: string, substr: string): string;
+    logExpression(base: string, operand: string): string;
 }

build/dialect/baseDialect.js

@@ -38,6 +38,9 @@ var SQLDialect = (function () {
     SQLDialect.prototype.booleanToSQL = function (bool) {
         return ('' + bool).toUpperCase();
     };
+    SQLDialect.prototype.floatDivision = function (numerator, denominator) {
+        return "(" + numerator + "/" + denominator + ")";
+    };
     SQLDialect.prototype.numberOrTimeToSQL = function (x) {
         if (x === null)
             return this.nullConstant();
@@ -94,7 +97,7 @@ var SQLDialect = (function () {
         return "(" + a + " IS NOT DISTINCT FROM " + b + ")";
     };
     SQLDialect.prototype.regexpExpression = function (expression, regexp) {
-        return "(" + expression + " REGEXP '" + regexp + "')";
+        return "(" + expression + " REGEXP " + this.escapeLiteral(regexp) + ")";
     };
     SQLDialect.prototype.inExpression = function (operand, start, end, bounds) {
         if (start === end && bounds === '[]')
@@ -117,6 +120,11 @@ var SQLDialect = (function () {
     SQLDialect.prototype.lengthExpression = function (a) {
         return "CHAR_LENGTH(" + a + ")";
     };
+    SQLDialect.prototype.logExpression = function (base, operand) {
+        if (base === String(Math.E))
+            return "LN(" + operand + ")";
+        return "LOG(" + base + "," + operand + ")";
+    };
     return SQLDialect;
 }());
 export { SQLDialect };

build/dialect/druidDialect.d.ts

@@ -2,24 +2,25 @@ import { Duration, Timezone } from 'chronoshift';
 import { PlyType } from '../types';
 import { SQLDialect } from './baseDialect';
 export declare class DruidDialect extends SQLDialect {
-    static TIME_BUCKETING: Record<string, string>;
     static TIME_PART_TO_FUNCTION: Record<string, string>;
     static CAST_TO_FUNCTION: Record<string, Record<string, string>>;
     constructor();
-    nullConstant(): string;
     dateToSQLDateString(date: Date): string;
+    floatDivision(numerator: string, denominator: string): string;
     constantGroupBy(): string;
     timeToSQL(date: Date): string;
     concatExpression(a: string, b: string): string;
     containsExpression(a: string, b: string): string;
-    coalesceExpression(a: string, b: string): string;
     substrExpression(a: string, position: number, length: number): string;
     isNotDistinctFromExpression(a: string, b: string): string;
     castExpression(inputType: PlyType, operand: string, cast: string): string;
+    private operandAsTimestamp;
     timeFloorExpression(operand: string, duration: Duration, timezone: Timezone): string;
     timeBucketExpression(operand: string, duration: Duration, timezone: Timezone): string;
     timePartExpression(operand: string, part: string, timezone: Timezone): string;
-    timeShiftExpression(operand: string, duration: Duration, timezone: Timezone): string;
+    timeShiftExpression(operand: string, duration: Duration, step: int, timezone: Timezone): string;
     extractExpression(operand: string, regexp: string): string;
+    regexpExpression(expression: string, regexp: string): string;
     indexOfExpression(str: string, substr: string): string;
+    logExpression(base: string, operand: string): string;
 }

build/dialect/druidDialect.js

@@ -5,15 +5,15 @@ var DruidDialect = (function (_super) {
     function DruidDialect() {
         return _super.call(this) || this;
     }
-    DruidDialect.prototype.nullConstant = function () {
-        return "''";
-    };
     DruidDialect.prototype.dateToSQLDateString = function (date) {
         return date.toISOString()
             .replace('T', ' ')
             .replace('Z', '')
             .replace(/\.000$/, '');
     };
+    DruidDialect.prototype.floatDivision = function (numerator, denominator) {
+        return "(" + numerator + "*1.0/" + denominator + ")";
+    };
     DruidDialect.prototype.constantGroupBy = function () {
         return "GROUP BY ''";
     };
@@ -28,13 +28,15 @@ var DruidDialect = (function (_super) {
     DruidDialect.prototype.containsExpression = function (a, b) {
         return "POSITION(" + a + " IN " + b + ")>0";
     };
-    DruidDialect.prototype.coalesceExpression = function (a, b) {
-        return "CASE WHEN " + a + "='' THEN " + b + " ELSE " + a + " END";
-    };
     DruidDialect.prototype.substrExpression = function (a, position, length) {
         return "SUBSTRING(" + a + "," + (position + 1) + "," + length + ")";
     };
     DruidDialect.prototype.isNotDistinctFromExpression = function (a, b) {
+        var nullConst = this.nullConstant();
+        if (a === nullConst)
+            return b + " IS " + nullConst;
+        if (b === nullConst)
+            return a + " IS " + nullConst;
         return "(" + a + "=" + b + ")";
     };
     DruidDialect.prototype.castExpression = function (inputType, operand, cast) {
@@ -43,11 +45,11 @@ var DruidDialect = (function (_super) {
             throw new Error("unsupported cast from " + inputType + " to " + cast + " in Druid dialect");
         return castFunction.replace(/\$\$/g, operand);
     };
+    DruidDialect.prototype.operandAsTimestamp = function (operand) {
+        return operand.includes('__time') ? operand : "TIME_PARSE(" + operand + ")";
+    };
     DruidDialect.prototype.timeFloorExpression = function (operand, duration, timezone) {
-        var bucketFormat = DruidDialect.TIME_BUCKETING[duration.toString()];
-        if (!bucketFormat)
-            throw new Error("unsupported duration '" + duration + "'");
-        return "FLOOR(" + operand + " TO " + bucketFormat + ")";
+        return "TIME_FLOOR(" + this.operandAsTimestamp(operand) + ", " + this.escapeLiteral(duration.toString()) + ", NULL, " + this.escapeLiteral(timezone.toString()) + ")";
     };
     DruidDialect.prototype.timeBucketExpression = function (operand, duration, timezone) {
         return this.timeFloorExpression(operand, duration, timezone);
@@ -56,66 +58,55 @@ var DruidDialect = (function (_super) {
         var timePartFunction = DruidDialect.TIME_PART_TO_FUNCTION[part];
         if (!timePartFunction)
             throw new Error("unsupported part " + part + " in Druid dialect");
-        return timePartFunction.replace(/\$\$/g, operand);
+        return timePartFunction
+            .replace(/\$\$/g, this.operandAsTimestamp(operand))
+            .replace(/##/g, this.escapeLiteral(timezone.toString()));
     };
-    DruidDialect.prototype.timeShiftExpression = function (operand, duration, timezone) {
-        var sqlFn = "DATE_ADD(";
-        var spans = duration.valueOf();
-        if (spans.week) {
-            return sqlFn + operand + ", INTERVAL " + String(spans.week) + ' WEEK)';
-        }
-        if (spans.year || spans.month) {
-            var expr = String(spans.year || 0) + "-" + String(spans.month || 0);
-            operand = sqlFn + operand + ", INTERVAL '" + expr + "' YEAR_MONTH)";
-        }
-        if (spans.day || spans.hour || spans.minute || spans.second) {
-            var expr = String(spans.day || 0) + " " + [spans.hour || 0, spans.minute || 0, spans.second || 0].join(':');
-            operand = sqlFn + operand + ", INTERVAL '" + expr + "' DAY_SECOND)";
-        }
-        return operand;
+    DruidDialect.prototype.timeShiftExpression = function (operand, duration, step, timezone) {
+        return "TIME_SHIFT(" + this.operandAsTimestamp(operand) + ", " + this.escapeLiteral(duration.toString()) + ", " + step + ", " + this.escapeLiteral(timezone.toString()) + ")";
     };
     DruidDialect.prototype.extractExpression = function (operand, regexp) {
-        return "(SELECT (REGEXP_MATCHES(" + operand + ", '" + regexp + "'))[1])";
+        return "REGEXP_EXTRACT(" + operand + ", " + this.escapeLiteral(regexp) + ", 1)";
+    };
+    DruidDialect.prototype.regexpExpression = function (expression, regexp) {
+        return "REGEXP_LIKE(" + expression + ", " + this.escapeLiteral(regexp) + ")";
     };
     DruidDialect.prototype.indexOfExpression = function (str, substr) {
         return "POSITION(" + substr + " IN " + str + ") - 1";
     };
-    DruidDialect.TIME_BUCKETING = {
-        "PT1S": "second",
-        "PT1M": "minute",
-        "PT1H": "hour",
-        "P1D": "day",
-        "P1W": "week",
-        "P1M": "month",
-        "P3M": "quarter",
-        "P1Y": "year"
+    DruidDialect.prototype.logExpression = function (base, operand) {
+        if (base === String(Math.E))
+            return "LN(" + operand + ")";
+        if (base === '10')
+            return "LOG10(" + operand + ")";
+        return "LN(" + operand + ")/LN(" + base + ")";
     };
     DruidDialect.TIME_PART_TO_FUNCTION = {
-        SECOND_OF_MINUTE: "EXTRACT(SECOND FROM $$)",
-        SECOND_OF_HOUR: "(EXTRACT(MINUTE FROM $$)*60+EXTRACT(SECOND FROM $$))",
-        SECOND_OF_DAY: "((EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$))*60+EXTRACT(SECOND FROM $$))",
-        SECOND_OF_WEEK: "(((MOD(CAST((TIME_EXTRACT($$,'DOW')+6) AS int),7)*24)+EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$))*60+EXTRACT(SECOND FROM $$))",
-        SECOND_OF_MONTH: "((((EXTRACT(DAY FROM $$)-1)*24)+EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$))*60+EXTRACT(SECOND FROM $$))",
-        SECOND_OF_YEAR: "((((TIME_EXTRACT($$,'DOY')-1)*24)+EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$))*60+EXTRACT(SECOND FROM $$))",
-        MINUTE_OF_HOUR: "EXTRACT(MINUTE FROM $$)",
-        MINUTE_OF_DAY: "EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$)",
-        MINUTE_OF_WEEK: "(MOD(CAST((TIME_EXTRACT($$,'DOW')+6) AS int),7)*24)+EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$)",
-        MINUTE_OF_MONTH: "((EXTRACT(DAY FROM $$)-1)*24)+EplyXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$)",
-        MINUTE_OF_YEAR: "((TIME_EXTRACT($$,'DOY')-1)*24)+EXTRACT(HOUR FROM $$)*60+EXTRACT(MINUTE FROM $$)",
-        HOUR_OF_DAY: "EXTRACT(HOUR FROM $$)",
-        HOUR_OF_WEEK: "(MOD(CAST((TIME_EXTRACT($$,'DOW')+6) AS int),7)*24+EXTRACT(HOUR FROM $$))",
-        HOUR_OF_MONTH: "((EXTRACT(DAY FROM $$)-1)*24+EXTRACT(HOUR FROM $$))",
-        HOUR_OF_YEAR: "((TIME_EXTRACT($$,'DOY')-1)*24+EXTRACT(HOUR FROM $$))",
-        DAY_OF_WEEK: "MOD(CAST((TIME_EXTRACT($$,'DOW')+6) AS int),7)+1",
-        DAY_OF_MONTH: "EXTRACT(DAY FROM $$)",
-        DAY_OF_YEAR: "TIME_EXTRACT($$,'DOY')",
-        WEEK_OF_YEAR: "TIME_EXTRACT($$,'WEEK')",
-        MONTH_OF_YEAR: "TIME_EXTRACT($$,'MONTH')",
-        YEAR: "EXTRACT(YEAR FROM $$)"
+        SECOND_OF_MINUTE: "TIME_EXTRACT($$,'SECOND',##)",
+        SECOND_OF_HOUR: "(TIME_EXTRACT($$,'MINUTE',##)*60+TIME_EXTRACT($$,'SECOND',##))",
+        SECOND_OF_DAY: "((TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##))*60+TIME_EXTRACT($$,'SECOND',##))",
+        SECOND_OF_WEEK: "(((MOD(CAST((TIME_EXTRACT($$,'DOW',##)+6) AS int),7)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##))*60+TIME_EXTRACT($$,'SECOND',##))",
+        SECOND_OF_MONTH: "((((TIME_EXTRACT($$,'DAY',##)-1)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##))*60+TIME_EXTRACT($$,'SECOND',##))",
+        SECOND_OF_YEAR: "((((TIME_EXTRACT($$,'DOY',##)-1)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##))*60+TIME_EXTRACT($$,'SECOND',##))",
+        MINUTE_OF_HOUR: "TIME_EXTRACT($$,'MINUTE',##)",
+        MINUTE_OF_DAY: "TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##)",
+        MINUTE_OF_WEEK: "(MOD(CAST((TIME_EXTRACT($$,'DOW',##)+6) AS int),7)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##)",
+        MINUTE_OF_MONTH: "((TIME_EXTRACT($$,'DAY',##)-1)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##)",
+        MINUTE_OF_YEAR: "((TIME_EXTRACT($$,'DOY',##)-1)*24)+TIME_EXTRACT($$,'HOUR',##)*60+TIME_EXTRACT($$,'MINUTE',##)",
+        HOUR_OF_DAY: "TIME_EXTRACT($$,'HOUR',##)",
+        HOUR_OF_WEEK: "(MOD(CAST((TIME_EXTRACT($$,'DOW',##)+6) AS int),7)*24+TIME_EXTRACT($$,'HOUR',##))",
+        HOUR_OF_MONTH: "((TIME_EXTRACT($$,'DAY',##)-1)*24+TIME_EXTRACT($$,'HOUR',##))",
+        HOUR_OF_YEAR: "((TIME_EXTRACT($$,'DOY',##)-1)*24+TIME_EXTRACT($$,'HOUR',##))",
+        DAY_OF_WEEK: "MOD(CAST((TIME_EXTRACT($$,'DOW',##)+6) AS int),7)+1",
+        DAY_OF_MONTH: "TIME_EXTRACT($$,'DAY',##)",
+        DAY_OF_YEAR: "TIME_EXTRACT($$,'DOY',##)",
+        WEEK_OF_YEAR: "TIME_EXTRACT($$,'WEEK',##)",
+        MONTH_OF_YEAR: "TIME_EXTRACT($$,'MONTH',##)",
+        YEAR: "TIME_EXTRACT($$,'YEAR',##)"
     };
     DruidDialect.CAST_TO_FUNCTION = {
         TIME: {
-            NUMBER: 'TO_TIMESTAMP($$::double precision / 1000)'
+            NUMBER: 'MILLIS_TO_TIMESTAMP(CAST($$ AS BIGINT))'
         },
         NUMBER: {
             TIME: "CAST($$ AS BIGINT)",

build/dialect/mySqlDialect.d.ts

@@ -22,7 +22,7 @@ export declare class MySQLDialect extends SQLDialect {
     timeFloorExpression(operand: string, duration: Duration, timezone: Timezone): string;
     timeBucketExpression(operand: string, duration: Duration, timezone: Timezone): string;
     timePartExpression(operand: string, part: string, timezone: Timezone): string;
-    timeShiftExpression(operand: string, duration: Duration, timezone: Timezone): string;
+    timeShiftExpression(operand: string, duration: Duration, step: int, timezone: Timezone): string;
     extractExpression(operand: string, regexp: string): string;
     indexOfExpression(str: string, substr: string): string;
 }