Fix NOT IN with NULL in the subquery or list to close #1414

src/15utility.js

@@ -1090,6 +1090,17 @@ var getValueOf = (utils.getValueOf = function (val) {
 		: val;
 });
 
+/**
+ * Result of `value NOT IN (set)` with SQL three-valued logic (#1414):
+ * false when the value is in the set, NULL (undefined) when it is not but the
+ * set contains NULL (null, undefined or NaN, as in the operator null check),
+ * otherwise true.
+ */
+utils.notIn = function (set, value) {
+	if (set.has(value)) return false;
+	return set.has(null) || set.has(undefined) || set.has(NaN) ? undefined : true;
+};
+
 /**
   Flat array by first row
   */

src/50expression.js

@@ -393,9 +393,10 @@
 						// Use a cache to avoid re-creating the Set on every identical query
 						alasql.sets = alasql.sets || {};
 						const allValues = this.right.map(value => value.value);
-						const allValuesStr = allValues.join(',');
+						// Key by type and value, so lists like (2, NULL) and (2, '') or (1) and ('1') don't share a Set
+						const allValuesStr = JSON.stringify(allValues.map(v => [typeof v, v]));
 						alasql.sets[allValuesStr] = alasql.sets[allValuesStr] || new Set(allValues);
-						s = `alasql.sets["${allValuesStr}"].has(alasql.utils.getValueOf(${leftJS()}))`;
+						s = `alasql.sets[${JSON.stringify(allValuesStr)}].has(alasql.utils.getValueOf(${leftJS()}))`;
 					}
 				} else {
 					s = `(${rightJS()}.indexOf(${leftJS()}) > -1)`;
@@ -406,8 +407,8 @@
 					// If correlated, we cannot cache the results as they depend on the current row
 					const cacheKey = `notIn${this.queriesidx}`;
 					const checkCorrelated = `(this.queriesfn[${this.queriesidx}].query && this.queriesfn[${this.queriesidx}].query.isCorrelated)`;
-					const cachedLookup = `(!(this.subqueryCache = this.subqueryCache || {}, this.subqueryCache.${cacheKey} || (this.subqueryCache.${cacheKey} = new Set(alasql.utils.flatArray(this.queriesfn[${this.queriesidx}](params, null, ${context})).map(alasql.utils.getValueOf)))).has(alasql.utils.getValueOf(${leftJS()})))`;
-					const uncachedLookup = `(alasql.utils.flatArray(this.queriesfn[${this.queriesidx}](params, null, ${context})).indexOf(alasql.utils.getValueOf(${leftJS()})) < 0)`;
+					const cachedLookup = `alasql.utils.notIn((this.subqueryCache = this.subqueryCache || {}, this.subqueryCache.${cacheKey} || (this.subqueryCache.${cacheKey} = new Set(alasql.utils.flatArray(this.queriesfn[${this.queriesidx}](params, null, ${context})).map(alasql.utils.getValueOf)))), alasql.utils.getValueOf(${leftJS()}))`;
+					const uncachedLookup = `alasql.utils.notIn(new Set(alasql.utils.flatArray(this.queriesfn[${this.queriesidx}](params, null, ${context})).map(alasql.utils.getValueOf)), alasql.utils.getValueOf(${leftJS()}))`;
 					s = `(${checkCorrelated} ? ${uncachedLookup} : ${cachedLookup})`;
 				} else if (Array.isArray(this.right)) {
 					// Empty array: everything is NOT IN an empty set, always true
@@ -422,14 +423,15 @@
 						this.right.some(value => value instanceof yy.ParamValue)
 					) {
 						// Leverage JS Set for faster lookups than arrays
-						s = `(!(new Set([${this.right.map(ref).join(',')}]).has(alasql.utils.getValueOf(${leftJS()}))))`;
+						s = `alasql.utils.notIn(new Set([${this.right.map(ref).join(',')}]), alasql.utils.getValueOf(${leftJS()}))`;
 					} else {
 						// Use a cache to avoid re-creating the Set on every identical query
 						alasql.sets = alasql.sets || {};
 						const allValues = this.right.map(value => value.value);
-						const allValuesStr = allValues.join(',');
+						// Key by type and value, so lists like (2, NULL) and (2, '') or (1) and ('1') don't share a Set
+						const allValuesStr = JSON.stringify(allValues.map(v => [typeof v, v]));
 						alasql.sets[allValuesStr] = alasql.sets[allValuesStr] || new Set(allValues);
-						s = `!alasql.sets["${allValuesStr}"].has(alasql.utils.getValueOf(${leftJS()}))`;
+						s = `alasql.utils.notIn(alasql.sets[${JSON.stringify(allValuesStr)}], alasql.utils.getValueOf(${leftJS()}))`;
 					}
 				} else {
 					s = `(${rightJS()}.indexOf(${leftJS()}) === -1)`;

test/test1414.js

@@ -0,0 +1,87 @@
+if (typeof exports === 'object') {
+	var assert = require('assert');
+	var alasql = require('..');
+}
+
+let testId = '1414'; // NOT IN with NULL in the subquery or list
+
+describe(`Test ${testId} - NOT IN with NULL values follows SQL semantics`, function () {
+	before(function () {
+		alasql('create database test' + testId);
+		alasql('use test' + testId);
+	});
+
+	after(function () {
+		alasql('drop database test' + testId);
+	});
+
+	it('A) Example from the issue', function () {
+		alasql('CREATE TABLE R (a number)');
+		alasql('CREATE TABLE S (b number)');
+		alasql.tables.R.data = [{a: 1}, {a: null}];
+		alasql.tables.S.data = [{b: null}];
+		const res = alasql('select a from R where a not in (select b from S)');
+		assert.deepStrictEqual(res, []);
+	});
+
+	const R = [{a: 1}, {a: 2}, {a: null}];
+
+	it('B) NOT IN subquery containing NULL and a value', function () {
+		const res = alasql('SELECT a FROM ? WHERE a NOT IN (SELECT b FROM ?)', [
+			R,
+			[{b: null}, {b: 2}],
+		]);
+		assert.deepStrictEqual(res, []);
+	});
+
+	it('C) Correlated NOT IN subquery containing NULL', function () {
+		alasql('CREATE TABLE R2 (id INT, a INT)');
+		alasql('CREATE TABLE S2 (id INT, b INT)');
+		alasql('INSERT INTO R2 VALUES (1, 1), (2, 1)');
+		alasql('INSERT INTO S2 VALUES (1, NULL), (2, 5)');
+		const res = alasql('SELECT id FROM R2 WHERE a NOT IN (SELECT b FROM S2 WHERE S2.id = R2.id)');
+		assert.deepStrictEqual(res, [{id: 2}]);
+	});
+
+	it('D) NOT IN list containing NULL', function () {
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a NOT IN (2, NULL)', [R]), []);
+	});
+
+	it('E) The value of NOT IN is NULL (undefined) when nothing matches and the set has NULL', function () {
+		assert.deepStrictEqual(alasql('SELECT 1 NOT IN (SELECT b FROM ?) AS x', [[{b: null}]]), [
+			{x: undefined},
+		]);
+		assert.deepStrictEqual(
+			alasql('SELECT 2 NOT IN (SELECT b FROM ?) AS x', [[{b: null}, {b: 2}]]),
+			[{x: false}]
+		);
+		assert.deepStrictEqual(alasql('SELECT 1 NOT IN (2, NULL) AS x'), [{x: undefined}]);
+	});
+
+	it('F) Without NULL in the set NOT IN and IN are unchanged', function () {
+		const S = [{b: 2}, {b: 3}];
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a NOT IN (SELECT b FROM ?)', [R, S]), [
+			{a: 1},
+		]);
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a NOT IN (2, 3)', [R]), [{a: 1}]);
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a IN (SELECT b FROM ?)', [R, S]), [
+			{a: 2},
+		]);
+		assert.deepStrictEqual(
+			alasql('SELECT a FROM ? WHERE a IN (SELECT b FROM ?)', [R, [{b: null}, {b: 2}]]),
+			[{a: 2}]
+		);
+	});
+
+	it('G) Cached value lists that only differ by NULL, empty string or type are kept apart', function () {
+		const rows = [{a: 1}, {a: 2}, {a: ''}, {a: '1'}];
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a IN (2, NULL)', [rows]), [{a: 2}]);
+		assert.deepStrictEqual(alasql("SELECT a FROM ? WHERE a NOT IN (2, '')", [rows]), [
+			{a: 1},
+			{a: '1'},
+		]);
+		assert.deepStrictEqual(alasql("SELECT a FROM ? WHERE a IN (2, '')", [rows]), [{a: 2}, {a: ''}]);
+		assert.deepStrictEqual(alasql('SELECT a FROM ? WHERE a IN (1, 3)', [rows]), [{a: 1}]);
+		assert.deepStrictEqual(alasql("SELECT a FROM ? WHERE a IN ('1', 3)", [rows]), [{a: '1'}]);
+	});
+});