From 2701feca2e0f0c352cb37c6358abac4d2eed1d96 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:02:31 -0700 Subject: [PATCH 001/141] Type inference experimentation. --- logica.py | 14 +++++++--- type_inference/research/infer.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 type_inference/research/infer.py diff --git a/logica.py b/logica.py index b9c5262c..d96de08e 100755 --- a/logica.py +++ b/logica.py @@ -47,6 +47,7 @@ from compiler import rule_translate from compiler import universe from parser_py import parse + from type_inference.research import infer else: from .common import color from .common import sqlite3_logica @@ -54,6 +55,7 @@ from .compiler import rule_translate from .compiler import universe from .parser_py import parse + from .type_inference.research import infer def ReadUserFlags(rules, argv): @@ -145,7 +147,7 @@ def main(argv): command = argv[2] - commands = ['parse', 'print', 'run', 'run_to_csv', 'run_in_terminal'] + commands = ['parse', 'print', 'run', 'run_to_csv', 'run_in_terminal', 'infer_types'] if command not in commands: print(color.Format('Unknown command {warning}{command}{end}. ' @@ -165,8 +167,14 @@ def main(argv): sys.exit(1) if command == 'parse': - # No indentation to avoid file size inflation. - print(json.dumps(parsed_rules, sort_keys=True, indent='')) + # Minimal indentation for better readability of deep objects. + print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) + return 0 + + if command == 'infer_types': + typing_engine = infer.TypesInferenceEngine() + typing_engine.InferTypes() + print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 predicates = argv[3] diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py new file mode 100644 index 00000000..bb9814f5 --- /dev/null +++ b/type_inference/research/infer.py @@ -0,0 +1,44 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def Walk(node, act): + """Walking over a dictionary of lists, acting on each element.""" + if isinstance(node, list): + for v in x: + Walk(v, act) + if isinstance(x, dict): + act(node) + for k in node: + Walk(node[k], act) + + +def ActInitializingTypes(node): + expression_fields = ['expression', 'left_hand_side', 'right_hand_side'] + for f in expression_fields: + if f in node: + node['expression']['type'] = 'Any' + + +class TypesInferenceEngine: + def __init__(self): + self.predicate_argumets_types = {} + + def InitTypes(parsed_rules): + for rule in parsed_rules: + Walk(rule, ActInitializingTypes) + + def InferTypes(parsed_rules): + self.InitTypes() From fbaca8a28b215fbb92c0c3209570df665e96ff17 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:06:41 -0700 Subject: [PATCH 002/141] Type inference experimentation. --- logica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logica.py b/logica.py index d96de08e..c7c27d37 100755 --- a/logica.py +++ b/logica.py @@ -132,7 +132,7 @@ def main(argv): 'GoodIdea(snack: "carrots")\'') return 1 - if len(argv) == 3 and argv[2] == 'parse': + if len(argv) == 3 and argv[2] in ['parse', 'infer_types']: pass # compile needs just 2 actual arguments. else: if len(argv) < 4: From 05203e7085332c6d1cd9d85c09485e16c3fc6afc Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:08:42 -0700 Subject: [PATCH 003/141] Type inference experimentation. --- type_inference/research/infer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index bb9814f5..309482c7 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -36,9 +36,9 @@ class TypesInferenceEngine: def __init__(self): self.predicate_argumets_types = {} - def InitTypes(parsed_rules): + def InitTypes(self, parsed_rules): for rule in parsed_rules: Walk(rule, ActInitializingTypes) - def InferTypes(parsed_rules): + def InferTypes(self, parsed_rules): self.InitTypes() From 9b659b98760ff03276e9238c6f57165866eb0642 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:09:39 -0700 Subject: [PATCH 004/141] Type inference experimentation. --- logica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logica.py b/logica.py index c7c27d37..171ddab5 100755 --- a/logica.py +++ b/logica.py @@ -173,7 +173,7 @@ def main(argv): if command == 'infer_types': typing_engine = infer.TypesInferenceEngine() - typing_engine.InferTypes() + typing_engine.InferTypes(parsed_rules) print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 From bf9483dd31104ab4babdb8dfce64d49413630645 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:11:13 -0700 Subject: [PATCH 005/141] Type inference experimentation. --- type_inference/research/infer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 309482c7..0ab9a4a5 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -33,12 +33,13 @@ def ActInitializingTypes(node): class TypesInferenceEngine: - def __init__(self): + def __init__(self, parsed_rules): + self.parsed_rules = parsed_rules self.predicate_argumets_types = {} - def InitTypes(self, parsed_rules): - for rule in parsed_rules: + def InitTypes(self): + for rule in self.parsed_rules: Walk(rule, ActInitializingTypes) - def InferTypes(self, parsed_rules): + def InferTypes(self): self.InitTypes() From d268e2845eab445e43ede3cb2c5cf6b7cad63780 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:12:24 -0700 Subject: [PATCH 006/141] Type inference experimentation. --- logica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logica.py b/logica.py index 171ddab5..e28d7d0f 100755 --- a/logica.py +++ b/logica.py @@ -172,7 +172,7 @@ def main(argv): return 0 if command == 'infer_types': - typing_engine = infer.TypesInferenceEngine() + typing_engine = infer.TypesInferenceEngine(parsed_rules) typing_engine.InferTypes(parsed_rules) print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 From 760050fadc1ef171a4f12839d83559cb950247d3 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:13:03 -0700 Subject: [PATCH 007/141] Type inference experimentation. --- logica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logica.py b/logica.py index e28d7d0f..c14ff6fc 100755 --- a/logica.py +++ b/logica.py @@ -173,7 +173,7 @@ def main(argv): if command == 'infer_types': typing_engine = infer.TypesInferenceEngine(parsed_rules) - typing_engine.InferTypes(parsed_rules) + typing_engine.InferTypes() print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 From 338ad4bb3d979e87a80863776e57109a6ab7a458 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:13:31 -0700 Subject: [PATCH 008/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 0ab9a4a5..764e44f7 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -17,7 +17,7 @@ def Walk(node, act): """Walking over a dictionary of lists, acting on each element.""" if isinstance(node, list): - for v in x: + for v in node: Walk(v, act) if isinstance(x, dict): act(node) From 62b1541fd877cd45e303728d8d60e5cdd9f16e19 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:13:59 -0700 Subject: [PATCH 009/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 764e44f7..96a25c8e 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -19,7 +19,7 @@ def Walk(node, act): if isinstance(node, list): for v in node: Walk(v, act) - if isinstance(x, dict): + if isinstance(node, dict): act(node) for k in node: Walk(node[k], act) From 0a67871cc6e2197dd645aaf73f38c8d840956bb8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:31:30 -0700 Subject: [PATCH 010/141] Type inference experimentation. --- type_inference/research/algebra.py | 134 +++++++++++++++++++++++++++++ type_inference/research/infer.py | 28 +++++- 2 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 type_inference/research/algebra.py diff --git a/type_inference/research/algebra.py b/type_inference/research/algebra.py new file mode 100644 index 00000000..de8f1d4b --- /dev/null +++ b/type_inference/research/algebra.py @@ -0,0 +1,134 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class OpenRecord(dict): + def __str__(self): + return '{%s, ...}' % str(dict(self))[1:-1] + def __repr__(self): + return str(self) + + +class ClosedRecord(dict): + def __str__(self): + return str(dict(self)) + def __repr__(self): + return str(self) + + +class BadType(tuple): + def __str__(self): + return f'({self[0]} is incompatible with {self[1]})' + def __repr__(self): + return str(self) + + +def Rank(x): + """Rank of the type, arbitrary order for sorting.""" + if isinstance(x, BadType): # Tuple means error. + return -1 + if x == 'Any': + return 0 + if x == 'Num': + return 1 + if x == 'Str': + return 2 + if isinstance(x, list): + return 3 + if isinstance(x, OpenRecord): + return 4 + if isinstance(x, ClosedRecord): + return 5 + assert False, 'Bad type: %s' % x + + +def Incompatible(a, b): + return BadType((a, b)) + + +def Intersect(a, b): + """Intersection of types a and b.""" + if isinstance(a, BadType) or isinstance(b, BadType): + return a + + if Rank(a) > Rank(b): + a, b = b, a + + if a == 'Any': + return b + + if a in ('Num', 'Str'): + if a == b: + return b + return Incompatible(a, b) # Type error: a is incompatible with b. + + if isinstance(a, list): + if isinstance(b, list): + a_element, b_element = a + b + new_element = Intersect(a_element, b_element) + if isinstance(new_element, BadType): + return Incompatible(a, b) + return [new_element] + return Incompatible(a, b) + + if isinstance(a, OpenRecord): + if isinstance(b, OpenRecord): + return IntersectFriendlyRecords(a, b, OpenRecord) + if isinstance(b, ClosedRecord): + if set(a) <= set(b): + return IntersectFriendlyRecords(a, b, ClosedRecord) + return Incompatible(a, b) + assert False + + if isinstance(a, ClosedRecord): + if isinstance(b, ClosedRecord): + if set(a) == set(b): + return IntersectFriendlyRecords(a, b, ClosedRecord) + return Incompatible(a, b) + assert False + assert False + +def IntersectFriendlyRecords(a, b, record_type): + """Intersecting records assuming that their fields are compatible.""" + result = {} + for f in set(a) | set(b): + x = Intersect(a.get(f, 'Any'), b.get(f, 'Any')) + if isinstance(x, BadType): # Ooops, field type error. + return Incompatible(a, b) + result[f] = x + return record_type(result) + +def IntersectListElement(a_list, b_element): + """Analysis of expression `b in a`.""" + a_result = Intersect(a_list, [b_element]) + if isinstance(a_result, BadType): + return (Incompatible(a_list, [b_element]), b_element) + + if isinstance(b_element, list): # No lists of lists in BigQuery. + return (a_result, Incompatible(b_element, 'NotAList')) + + [a_element] = a_result + b_result = Intersect(b_element, a_element) + return (a_result, b_result) + +def IntersectRecordField(a_record, field_name, b_field_value): + """Analysis of expresson `a.f = b`.""" + a_result = Intersect(a_record, OpenRecord({field_name: b_field_value})) + if isinstance(a_result, BadType): + return (a_result, b_field_value) + b_result = Intersect(a_result[field_name], b_field_value) + return (a_result, b_result) + diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 96a25c8e..d018cd82 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -14,6 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +if '.' not in __package__: + from type_inference.research import algebra +else: + from ..type_inference.research import algebra + +def ExpressionFields(): + return ['expression', 'left_hand_side', 'right_hand_side'] + + def Walk(node, act): """Walking over a dictionary of lists, acting on each element.""" if isinstance(node, list): @@ -26,12 +35,20 @@ def Walk(node, act): def ActInitializingTypes(node): - expression_fields = ['expression', 'left_hand_side', 'right_hand_side'] - for f in expression_fields: + for f in ExpressionFields(): if f in node: - node['expression']['type'] = 'Any' + node[f]['type'] = 'Any' +def ActMindingPodLiterals(node): + for f in ExpressionFields(): + if f in node: + if 'literal' in node['f']: + if 'the_number' in node['f']['literal']: + node[f]['type'] = algebra.Intersect(node[f]['type'], 'Num') + if 'the_string' in node['f']['literal']: + node[f]['type'] = algebra.Intersect(node[f]['type'], 'Str') + class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules @@ -41,5 +58,10 @@ def InitTypes(self): for rule in self.parsed_rules: Walk(rule, ActInitializingTypes) + def MindPodLiterals(self): + for rule in self.parsed_rules: + Walk(rule, ActMindingPodLiterals) + def InferTypes(self): self.InitTypes() + From e5de6976fe7ff2b8d06f1ddd544fba73cfb2e3aa Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:32:04 -0700 Subject: [PATCH 011/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index d018cd82..4807474c 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -17,7 +17,7 @@ if '.' not in __package__: from type_inference.research import algebra else: - from ..type_inference.research import algebra + from ...type_inference.research import algebra def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] From 06fd1d7f6142b80bf7bb44421c9ac1619803684a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:32:43 -0700 Subject: [PATCH 012/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 4807474c..d018cd82 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -17,7 +17,7 @@ if '.' not in __package__: from type_inference.research import algebra else: - from ...type_inference.research import algebra + from ..type_inference.research import algebra def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] From b60791295d190431e904c7459b1684b141327563 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:34:34 -0700 Subject: [PATCH 013/141] Type inference experimentation. --- type_inference/research/infer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index d018cd82..b99884a5 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +print('>>', __package__) + if '.' not in __package__: from type_inference.research import algebra else: From 2899e8cca405391f4d606ebe2a13fea200592b1b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:36:46 -0700 Subject: [PATCH 014/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index b99884a5..632facb8 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -19,7 +19,7 @@ if '.' not in __package__: from type_inference.research import algebra else: - from ..type_inference.research import algebra + import algebra def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] From 967f39d33d0f168155618198b1671a9597fe73ef Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:37:34 -0700 Subject: [PATCH 015/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 632facb8..e66ea66c 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -19,7 +19,7 @@ if '.' not in __package__: from type_inference.research import algebra else: - import algebra + from ..research import algebra def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] From dbb4d0810882f0ae3cf1c59f64c2d2cb08d68a5d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:38:08 -0700 Subject: [PATCH 016/141] Type inference experimentation. --- type_inference/research/infer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index e66ea66c..c6969f77 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -print('>>', __package__) - if '.' not in __package__: from type_inference.research import algebra else: @@ -66,4 +64,5 @@ def MindPodLiterals(self): def InferTypes(self): self.InitTypes() + self.MindPodLiterals() From 2410469e98075ff132cea038d76426c9d62e0d37 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 19:38:37 -0700 Subject: [PATCH 017/141] Type inference experimentation. --- type_inference/research/infer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index c6969f77..f2daa805 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -43,12 +43,13 @@ def ActInitializingTypes(node): def ActMindingPodLiterals(node): for f in ExpressionFields(): if f in node: - if 'literal' in node['f']: - if 'the_number' in node['f']['literal']: + if 'literal' in node[f]: + if 'the_number' in node[f]['literal']: node[f]['type'] = algebra.Intersect(node[f]['type'], 'Num') - if 'the_string' in node['f']['literal']: + if 'the_string' in node[f]['literal']: node[f]['type'] = algebra.Intersect(node[f]['type'], 'Str') + class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules From 00d8472baea06d754c7cba1ba21a9d054a981578 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:05:15 -0700 Subject: [PATCH 018/141] Type inference experimentation. --- type_inference/research/infer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index f2daa805..6736184e 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -37,7 +37,7 @@ def Walk(node, act): def ActInitializingTypes(node): for f in ExpressionFields(): if f in node: - node[f]['type'] = 'Any' + node[f]['type'] = {'the_type': 'Any'} def ActMindingPodLiterals(node): @@ -45,9 +45,9 @@ def ActMindingPodLiterals(node): if f in node: if 'literal' in node[f]: if 'the_number' in node[f]['literal']: - node[f]['type'] = algebra.Intersect(node[f]['type'], 'Num') + node[f]['type']['the_type'] = algebra.Intersect(node[f]['type']['the_type'], 'Num') if 'the_string' in node[f]['literal']: - node[f]['type'] = algebra.Intersect(node[f]['type'], 'Str') + node[f]['type']['the_type'] = algebra.Intersect(node[f]['type']['the_type'], 'Str') class TypesInferenceEngine: From 96f16595e647952fc0739f89c717cec95283f1b8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:12:44 -0700 Subject: [PATCH 019/141] Type inference experimentation. --- type_inference/research/infer.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 6736184e..b26944fb 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -34,12 +34,6 @@ def Walk(node, act): Walk(node[k], act) -def ActInitializingTypes(node): - for f in ExpressionFields(): - if f in node: - node[f]['type'] = {'the_type': 'Any'} - - def ActMindingPodLiterals(node): for f in ExpressionFields(): if f in node: @@ -54,10 +48,20 @@ class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules self.predicate_argumets_types = {} + self.variable_type = {} + + def ActInitializingTypes(self, node): + for f in ExpressionFields(): + if f in node: + if 'variable' in node[f]: + use_type = self.variable_type.get(node[f]['variable']['var_name'], {'the_type': 'Any'}) + else: + use_type = {'the_type': 'Any'} + node[f]['type'] = use_type def InitTypes(self): for rule in self.parsed_rules: - Walk(rule, ActInitializingTypes) + Walk(rule, self.ActInitializingTypes) def MindPodLiterals(self): for rule in self.parsed_rules: From 3f7b4672e21636fc9f133e546c0ae82c551944dd Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:25:26 -0700 Subject: [PATCH 020/141] Type inference experimentation. --- type_inference/research/infer.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index b26944fb..1df40c2d 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -70,4 +70,29 @@ def MindPodLiterals(self): def InferTypes(self): self.InitTypes() self.MindPodLiterals() - + for rule in self.parsed_rules: + TypeInferenceForRule(rule) + + +class TypeInferenceForRule: + def __init__(self, rule): + self.rule = rule + self.inference_complete = False + self.IterateInference(self) + + def ActUnifying(self, node): + if 'unification' in node: + left_type = node['unification']['left_hand_side']['type']['the_type'] + right_type = node['unification']['right_hand_side']['type']['the_type'] + new_type = algebra.Intersect(left_type, right_type) + if new_type != left_type: + self.inference_complete = False + node['unification']['left_hand_side']['type']['the_type'] = new_type + if new_type != right_type: + self.inference_complete = False + node['unification']['right_hand_side']['type']['the_type'] = new_type + + def IterateInference(self): + while not self.inference_complete: + self.inference_complete = True + Walk(self.rule, self.ActUnifying) From c1312a7542663e464188e343be925908adf3d7af Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:25:54 -0700 Subject: [PATCH 021/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 1df40c2d..ba7edffd 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -78,7 +78,7 @@ class TypeInferenceForRule: def __init__(self, rule): self.rule = rule self.inference_complete = False - self.IterateInference(self) + self.IterateInference() def ActUnifying(self, node): if 'unification' in node: From 48f6d0b7236a2f7a061fe57aff329ede7143716f Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:28:15 -0700 Subject: [PATCH 022/141] Type inference experimentation. --- type_inference/research/infer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index ba7edffd..8609881a 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -51,12 +51,16 @@ def __init__(self, parsed_rules): self.variable_type = {} def ActInitializingTypes(self, node): + i = 0 for f in ExpressionFields(): if f in node: + i += 1 if 'variable' in node[f]: - use_type = self.variable_type.get(node[f]['variable']['var_name'], {'the_type': 'Any'}) + use_type = self.variable_type.get( + node[f]['variable']['var_name'], + {'the_type': 'Any', 'type_id': i}) else: - use_type = {'the_type': 'Any'} + use_type = {'the_type': 'Any', 'type_id': i} node[f]['type'] = use_type def InitTypes(self): From 5cd7461203c2a1ab9519f90198837d25ebc118f5 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:29:55 -0700 Subject: [PATCH 023/141] Type inference experimentation. --- type_inference/research/infer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 8609881a..e5e30f04 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -49,12 +49,17 @@ def __init__(self, parsed_rules): self.parsed_rules = parsed_rules self.predicate_argumets_types = {} self.variable_type = {} + self.type_id_counter = 0 + + def GetTypeId(self): + result = self.type_id_counter + self.type_id_counter += 1 + return result def ActInitializingTypes(self, node): - i = 0 for f in ExpressionFields(): if f in node: - i += 1 + i = self.GetTypeId() if 'variable' in node[f]: use_type = self.variable_type.get( node[f]['variable']['var_name'], From ae7673fb55b44281f0d612901e1f55cae927e67b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:32:42 -0700 Subject: [PATCH 024/141] Type inference experimentation. --- type_inference/research/infer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index e5e30f04..b4ef0553 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -61,9 +61,11 @@ def ActInitializingTypes(self, node): if f in node: i = self.GetTypeId() if 'variable' in node[f]: + var_name = node[f]['variable']['var_name'] use_type = self.variable_type.get( - node[f]['variable']['var_name'], + var_name, {'the_type': 'Any', 'type_id': i}) + self.variable_type[var_name] = use_type else: use_type = {'the_type': 'Any', 'type_id': i} node[f]['type'] = use_type From e9e55bf21bd962884af1e25bce02511315e66ba4 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:52:53 -0700 Subject: [PATCH 025/141] Type inference experimentation. --- type_inference/research/infer.py | 19 ++++++++++++ type_inference/research/types_of_builtins.py | 32 ++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 type_inference/research/types_of_builtins.py diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index b4ef0553..8e2b666b 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -16,8 +16,11 @@ if '.' not in __package__: from type_inference.research import algebra + from type_inference.research import types_of_builtins else: from ..research import algebra + from ..research import types_of_builtins + def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] @@ -50,6 +53,7 @@ def __init__(self, parsed_rules): self.predicate_argumets_types = {} self.variable_type = {} self.type_id_counter = 0 + self.types_of_builtins = types_of_builtins.TypesOfBultins() def GetTypeId(self): result = self.type_id_counter @@ -78,9 +82,24 @@ def MindPodLiterals(self): for rule in self.parsed_rules: Walk(rule, ActMindingPodLiterals) + def ActMindingBuiltinFieldTypes(self): + for f in ExpressionFields(): + if f in node: + e = node[f] + if 'call' in e: + p = e['call']['predicate_name'] + t = e['type']['the_type'] + if p in self.types_of_builtins: + e['type']['the_type'] = algebra.Intersect(t, self.types_of_builtins[p]) + + def MindBuiltinFieldTypes(self): + for rule in self.parsed_rules: + Walk(rule, ActMindingBuiltinFieldTypes) + def InferTypes(self): self.InitTypes() self.MindPodLiterals() + self.MindBuiltinFieldTypes() for rule in self.parsed_rules: TypeInferenceForRule(rule) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py new file mode 100644 index 00000000..2d78deef --- /dev/null +++ b/type_inference/research/types_of_builtins.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def TypesOfBultins(): + return { + '+': { + 'left': 'Num', + 'right': 'Num', + 'logica_value': 'Num' + }, + 'Num': { + 'col0': 'Num', + 'logica_value': 'Num' + }, + 'Str': { + 'col0': 'Str', + 'logica_value': 'Str' + } + } \ No newline at end of file From dbb2926bfb729c5e0c0635ff8d721e7732c680c4 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:55:23 -0700 Subject: [PATCH 026/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 8e2b666b..5421122c 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -94,7 +94,7 @@ def ActMindingBuiltinFieldTypes(self): def MindBuiltinFieldTypes(self): for rule in self.parsed_rules: - Walk(rule, ActMindingBuiltinFieldTypes) + Walk(rule, self.ActMindingBuiltinFieldTypes) def InferTypes(self): self.InitTypes() From 316b20bd6acb6de8c2f37a7f8b8f30f1a152831f Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:55:42 -0700 Subject: [PATCH 027/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 5421122c..b3c8a657 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -82,7 +82,7 @@ def MindPodLiterals(self): for rule in self.parsed_rules: Walk(rule, ActMindingPodLiterals) - def ActMindingBuiltinFieldTypes(self): + def ActMindingBuiltinFieldTypes(self, node): for f in ExpressionFields(): if f in node: e = node[f] From f1af45aa505a4ec3fb706d33b89db23b79392172 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 22:56:21 -0700 Subject: [PATCH 028/141] Type inference experimentation. --- type_inference/research/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index b3c8a657..58985039 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -90,7 +90,7 @@ def ActMindingBuiltinFieldTypes(self, node): p = e['call']['predicate_name'] t = e['type']['the_type'] if p in self.types_of_builtins: - e['type']['the_type'] = algebra.Intersect(t, self.types_of_builtins[p]) + e['type']['the_type'] = algebra.Intersect(t, self.types_of_builtins[p]['logica_value']) def MindBuiltinFieldTypes(self): for rule in self.parsed_rules: From 0965a4a2c7f93a5bea483120820d20a84e82a3f7 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 2 Jul 2023 23:02:32 -0700 Subject: [PATCH 029/141] Type inference experimentation. --- type_inference/research/infer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 58985039..5698cd76 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -88,9 +88,15 @@ def ActMindingBuiltinFieldTypes(self, node): e = node[f] if 'call' in e: p = e['call']['predicate_name'] - t = e['type']['the_type'] if p in self.types_of_builtins: - e['type']['the_type'] = algebra.Intersect(t, self.types_of_builtins[p]['logica_value']) + e['type']['the_type'] = algebra.Intersect( + e['type']['the_type'], + self.types_of_builtins[p]['logica_value']) + for fv in e['call']['record']['field_value']: + if fv['field'] in self.types_of_builtins[p]: + fv['value']['expression']['type']['the_type'] = algebra.Intersect( + fv['value']['expression']['type']['the_type'], + self.types_of_builtins[p][fv['field']]) def MindBuiltinFieldTypes(self): for rule in self.parsed_rules: From 65d9b254914763cc383c71454963fc11b63daa42 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 07:38:05 -0700 Subject: [PATCH 030/141] Type inference experimentation. --- type_inference/research/infer.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 5698cd76..097207f5 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -25,6 +25,11 @@ def ExpressionFields(): return ['expression', 'left_hand_side', 'right_hand_side'] +def ExpressionsIterator(node): + for f in ExpressionFields(): + if f in node: + yield node[f] + def Walk(node, act): """Walking over a dictionary of lists, acting on each element.""" @@ -61,18 +66,17 @@ def GetTypeId(self): return result def ActInitializingTypes(self, node): - for f in ExpressionFields(): - if f in node: - i = self.GetTypeId() - if 'variable' in node[f]: - var_name = node[f]['variable']['var_name'] - use_type = self.variable_type.get( - var_name, - {'the_type': 'Any', 'type_id': i}) - self.variable_type[var_name] = use_type - else: - use_type = {'the_type': 'Any', 'type_id': i} - node[f]['type'] = use_type + for e in ExpressionsIterator(node): + i = self.GetTypeId() + if 'variable' in e: + var_name = e['variable']['var_name'] + use_type = self.variable_type.get( + var_name, + {'the_type': 'Any', 'type_id': i}) + self.variable_type[var_name] = use_type + else: + use_type = {'the_type': 'Any', 'type_id': i} + e['type'] = use_type def InitTypes(self): for rule in self.parsed_rules: From d7b43619ac92ac31d6ad50bfa1814f8521b82cec Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 07:39:29 -0700 Subject: [PATCH 031/141] Type inference experimentation. --- type_inference/research/infer.py | 39 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 097207f5..a903d8da 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -43,13 +43,12 @@ def Walk(node, act): def ActMindingPodLiterals(node): - for f in ExpressionFields(): - if f in node: - if 'literal' in node[f]: - if 'the_number' in node[f]['literal']: - node[f]['type']['the_type'] = algebra.Intersect(node[f]['type']['the_type'], 'Num') - if 'the_string' in node[f]['literal']: - node[f]['type']['the_type'] = algebra.Intersect(node[f]['type']['the_type'], 'Str') + for e in ExpressionsIterator(node): + if 'literal' in e: + if 'the_number' in e['literal']: + e['type']['the_type'] = algebra.Intersect(e['type']['the_type'], 'Num') + if 'the_string' in e['literal']: + e['type']['the_type'] = algebra.Intersect(e['type']['the_type'], 'Str') class TypesInferenceEngine: @@ -87,20 +86,18 @@ def MindPodLiterals(self): Walk(rule, ActMindingPodLiterals) def ActMindingBuiltinFieldTypes(self, node): - for f in ExpressionFields(): - if f in node: - e = node[f] - if 'call' in e: - p = e['call']['predicate_name'] - if p in self.types_of_builtins: - e['type']['the_type'] = algebra.Intersect( - e['type']['the_type'], - self.types_of_builtins[p]['logica_value']) - for fv in e['call']['record']['field_value']: - if fv['field'] in self.types_of_builtins[p]: - fv['value']['expression']['type']['the_type'] = algebra.Intersect( - fv['value']['expression']['type']['the_type'], - self.types_of_builtins[p][fv['field']]) + for e in ExpressionsIterator(node): + if 'call' in e: + p = e['call']['predicate_name'] + if p in self.types_of_builtins: + e['type']['the_type'] = algebra.Intersect( + e['type']['the_type'], + self.types_of_builtins[p]['logica_value']) + for fv in e['call']['record']['field_value']: + if fv['field'] in self.types_of_builtins[p]: + fv['value']['expression']['type']['the_type'] = algebra.Intersect( + fv['value']['expression']['type']['the_type'], + self.types_of_builtins[p][fv['field']]) def MindBuiltinFieldTypes(self): for rule in self.parsed_rules: From ea6ca8132653b4d890b6eb81bf3395793d42d94c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 07:44:45 -0700 Subject: [PATCH 032/141] Type inference experimentation. --- type_inference/research/infer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index a903d8da..a26fbcba 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -29,6 +29,8 @@ def ExpressionsIterator(node): for f in ExpressionFields(): if f in node: yield node[f] + if 'record' in node and 'field_value' not in node['record']: + yield node['record'] def Walk(node, act): From 6f7ea927647aace080a3378843e64626eb00bbe6 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 08:05:52 -0700 Subject: [PATCH 033/141] Type inference experimentation. --- type_inference/research/infer.py | 42 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index a26fbcba..090b62f9 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -57,10 +57,27 @@ class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules self.predicate_argumets_types = {} + + def InferTypes(self): + for rule in self.parsed_rules: + t = TypeInferenceForRule(rule) + t.PerformInference() + + +class TypeInferenceForRule: + def __init__(self, rule): + self.rule = rule + self.inference_complete = False self.variable_type = {} self.type_id_counter = 0 self.types_of_builtins = types_of_builtins.TypesOfBultins() - + + def PerformInference(self): + self.InitTypes() + self.MindPodLiterals() + self.MindBuiltinFieldTypes() + self.IterateInference() + def GetTypeId(self): result = self.type_id_counter self.type_id_counter += 1 @@ -80,12 +97,10 @@ def ActInitializingTypes(self, node): e['type'] = use_type def InitTypes(self): - for rule in self.parsed_rules: - Walk(rule, self.ActInitializingTypes) + Walk(self.rule, self.ActInitializingTypes) def MindPodLiterals(self): - for rule in self.parsed_rules: - Walk(rule, ActMindingPodLiterals) + Walk(self.rule, ActMindingPodLiterals) def ActMindingBuiltinFieldTypes(self, node): for e in ExpressionsIterator(node): @@ -102,22 +117,7 @@ def ActMindingBuiltinFieldTypes(self, node): self.types_of_builtins[p][fv['field']]) def MindBuiltinFieldTypes(self): - for rule in self.parsed_rules: - Walk(rule, self.ActMindingBuiltinFieldTypes) - - def InferTypes(self): - self.InitTypes() - self.MindPodLiterals() - self.MindBuiltinFieldTypes() - for rule in self.parsed_rules: - TypeInferenceForRule(rule) - - -class TypeInferenceForRule: - def __init__(self, rule): - self.rule = rule - self.inference_complete = False - self.IterateInference() + Walk(self.rule, self.ActMindingBuiltinFieldTypes) def ActUnifying(self, node): if 'unification' in node: From 487d0d667928696f2f642b3d1dc893b7844f5965 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 08:18:58 -0700 Subject: [PATCH 034/141] Type inference experimentation. --- type_inference/research/infer.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 090b62f9..76002b76 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -130,8 +130,24 @@ def ActUnifying(self, node): if new_type != right_type: self.inference_complete = False node['unification']['right_hand_side']['type']['the_type'] = new_type - + + def ActUnderstandingSubscription(self, node): + if 'subscript' in node and 'record' in node['subscript']: + record_type = node['subscript']['record']['type']['the_type'] + field_type = node['type']['the_type'] + field_name = node['subscript']['subscript']['literal']['the_symbol']['symbol'] + new_record_type, new_field_type = algebra.IntersectRecordField( + record_type, field_name, field_type) + if new_record_type != record_type: + self.inference_complete = False + node['subscript']['record']['type']['the_type'] = new_record_type + if new_field_type != field_type: + self.inference_complete = False + node['type']['the_type'] = new_field_type + + def IterateInference(self): while not self.inference_complete: self.inference_complete = True Walk(self.rule, self.ActUnifying) + Walk(self.rule, self.ActUnderstandingSubscription) From 43c6d40a0ac7ecdd3d5642f051908f1a570d421c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 08:24:20 -0700 Subject: [PATCH 035/141] Type inference experimentation. --- type_inference/research/infer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 76002b76..65f71841 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -41,7 +41,8 @@ def Walk(node, act): if isinstance(node, dict): act(node) for k in node: - Walk(node[k], act) + if k != 'type': + Walk(node[k], act) def ActMindingPodLiterals(node): From ad1c78ba29aa0d7d8ee23b5f843a70ed91d6c92a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 3 Jul 2023 08:37:21 -0700 Subject: [PATCH 036/141] Type inference experimentation. --- type_inference/research/infer.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 65f71841..c3712756 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -146,9 +146,24 @@ def ActUnderstandingSubscription(self, node): self.inference_complete = False node['type']['the_type'] = new_field_type + def ActMindingRecordLiterals(self, node): + if 'type' in node and 'record' in node: + for fv in node['record']['field_value']: + record_type = node['type']['the_type'] + field_type = fv['value']['expression']['type']['the_type'] + field_name = fv['field'] + new_record_type, new_field_type = algebra.IntersectRecordField( + record_type, field_name, field_type) + if new_record_type != record_type: + self.inference_complete = False + node['type']['the_type'] = new_record_type + if new_field_type != field_type: + self.inference_complete = False + fv['value']['expression']['type']['the_type'] = new_field_type def IterateInference(self): while not self.inference_complete: self.inference_complete = True Walk(self.rule, self.ActUnifying) Walk(self.rule, self.ActUnderstandingSubscription) + Walk(self.rule, self.ActMindingRecordLiterals) From 203ed32a73f1ee3b05d03a354422ed66b190744e Mon Sep 17 00:00:00 2001 From: EvgSkv <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:02:50 -0700 Subject: [PATCH 037/141] Type inference experimentation. --- .gitignore | 1 + common/logica_test.py | 46 +++ run_all_tests.py | 2 + .../research/integration_tests/run_tests.py | 32 ++ .../integration_tests/typing_basic_test.l | 18 + .../integration_tests/typing_basic_test.txt | 384 ++++++++++++++++++ 6 files changed, 483 insertions(+) create mode 100644 type_inference/research/integration_tests/run_tests.py create mode 100644 type_inference/research/integration_tests/typing_basic_test.l create mode 100644 type_inference/research/integration_tests/typing_basic_test.txt diff --git a/.gitignore b/.gitignore index b3dbe982..84ada139 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ docs/.DS_Store +*.pyc diff --git a/common/logica_test.py b/common/logica_test.py index 987027ec..ee04ac2f 100644 --- a/common/logica_test.py +++ b/common/logica_test.py @@ -18,13 +18,18 @@ """Utilities for YotaQL tests.""" import subprocess +import json if '.' not in __package__: from common import color from common import logica_lib + from type_inference.research import infer + from parser_py import parse else: from ..common import color from ..common import logica_lib + from ..type_inference.research import infer + from ..parser_py import parse class TestManager(object): @@ -54,6 +59,47 @@ def RunTest(cls, name, src, predicate, golden, user_flags, cls.GOLDEN_RUN, cls.ANNOUNCE_TESTS, import_root) + @classmethod + def RunTypesTest(cls, name, src=None, golden=None): + if cls.RUN_ONLY and name not in cls.RUN_ONLY: + return + RunTypesTest(name, src, golden, + overwrite=cls.GOLDEN_RUN) + + +def RunTypesTest(name, src=None, golden=None, + overwrite=False): + src = src or (name + '.l') + golden = golden or (name + '.txt') + + test_result = '{warning}RUNNING{end}' + print(color.Format('% 50s %s' % (name, test_result))) + + program_text = open(src).read() + try: + parsed_rules = parse.ParseFile(program_text)['rule'] + except parse.ParsingException as parsing_exception: + parsing_exception.ShowMessage() + sys.exit(1) + + typing_engine = infer.TypesInferenceEngine(parsed_rules) + typing_engine.InferTypes() + result = json.dumps(parsed_rules, sort_keys=True, indent=' ') + + if overwrite: + with open(golden, 'w') as w: + w.write(result) + golden_result = open(golden).read() + + if result == golden_result: + test_result = '{ok}PASSED{end}' + else: + p = subprocess.Popen(['diff', '-', golden], stdin=subprocess.PIPE) + p.communicate(result.encode()) + test_result = '{error}FAILED{end}' + + print('\033[F\033[K' + color.Format('% 50s %s' % (name, test_result))) + def RunTest(name, src, predicate, golden, user_flags=None, diff --git a/run_all_tests.py b/run_all_tests.py index d51462f8..76a73460 100755 --- a/run_all_tests.py +++ b/run_all_tests.py @@ -21,6 +21,7 @@ from common import logica_test from integration_tests import run_tests as integration_tests from integration_tests.import_tests import run_tests as import_tests +from type_inference.research.integration_tests import run_tests as type_inference_tests if 'golden_run' in sys.argv: @@ -35,5 +36,6 @@ logica_test.PrintHeader() +type_inference_tests.RunAll() integration_tests.RunAll() import_tests.RunAll() diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py new file mode 100644 index 00000000..e805b84b --- /dev/null +++ b/type_inference/research/integration_tests/run_tests.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A suite of tests for type inference.""" + +from common import logica_test + +def RunTypesTest(name, src=None, golden=None): + src = src or ( + 'type_inference/research/integration_tests/' + name + '.l') + golden = golden or ( + 'type_inference/research/integration_tests/' + name + '.txt') + logica_test.TestManager.RunTypesTest(name, src, golden) + +def RunAll(): + RunTypesTest('typing_basic_test') + +if __name__ == '__main__': + RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_basic_test.l b/type_inference/research/integration_tests/typing_basic_test.l new file mode 100644 index 00000000..e23f08af --- /dev/null +++ b/type_inference/research/integration_tests/typing_basic_test.l @@ -0,0 +1,18 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Q(1, "a", x, z, w.a, {t: 1, z: "a", r: {z: 1}}) :- + x == 1 + z, T(z, w, l), e in l, w.a.b == 7; \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt new file mode 100644 index 00000000..46c0389c --- /dev/null +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -0,0 +1,384 @@ +[ + { + "body": { + "conjunction": { + "conjunct": [ + { + "unification": { + "left_hand_side": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "x" + } + }, + "right_hand_side": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 13 + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 3 + }, + "variable": { + "var_name": "z" + } + } + } + } + ] + } + }, + "type": { + "the_type": "Num", + "type_id": 12 + } + } + } + }, + { + "predicate": { + "predicate_name": "T", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 3 + }, + "variable": { + "var_name": "z" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "type": { + "the_type": { + "a": { + "b": "Num" + } + }, + "type_id": 5 + }, + "variable": { + "var_name": "w" + } + } + } + }, + { + "field": 2, + "value": { + "expression": { + "type": { + "the_type": "Any", + "type_id": 17 + }, + "variable": { + "var_name": "l" + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "variable": { + "var_name": "e" + } + }, + "list": { + "variable": { + "var_name": "l" + } + } + } + }, + { + "unification": { + "left_hand_side": { + "subscript": { + "record": { + "subscript": { + "record": { + "type": { + "the_type": { + "a": { + "b": "Num" + } + }, + "type_id": 5 + }, + "variable": { + "var_name": "w" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "a" + } + } + } + }, + "type": { + "the_type": { + "b": "Num" + }, + "type_id": 20 + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "b" + } + } + } + }, + "type": { + "the_type": "Num", + "type_id": 18 + } + }, + "right_hand_side": { + "literal": { + "the_number": { + "number": "7" + } + }, + "type": { + "the_type": "Num", + "type_id": 19 + } + } + } + } + ] + } + }, + "full_text": "Q(1, \"a\", x, z, w.a, {t: 1, z: \"a\", r: {z: 1}}) :-\n x == 1 + z, T(z, w, l), e in l, w.a.b == 7", + "head": { + "predicate_name": "Q", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 0 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "literal": { + "the_string": { + "the_string": "a" + } + }, + "type": { + "the_type": "Str", + "type_id": 1 + } + } + } + }, + { + "field": 2, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "x" + } + } + } + }, + { + "field": 3, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 3 + }, + "variable": { + "var_name": "z" + } + } + } + }, + { + "field": 4, + "value": { + "expression": { + "subscript": { + "record": { + "type": { + "the_type": { + "a": { + "b": "Num" + } + }, + "type_id": 5 + }, + "variable": { + "var_name": "w" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "a" + } + } + } + }, + "type": { + "the_type": { + "b": "Num" + }, + "type_id": 4 + } + } + } + }, + { + "field": 5, + "value": { + "expression": { + "record": { + "field_value": [ + { + "field": "t", + "value": { + "expression": { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + }, + { + "field": "z", + "value": { + "expression": { + "literal": { + "the_string": { + "the_string": "a" + } + }, + "type": { + "the_type": "Str", + "type_id": 8 + } + } + } + }, + { + "field": "r", + "value": { + "expression": { + "record": { + "field_value": [ + { + "field": "z", + "value": { + "expression": { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 10 + } + } + } + } + ] + }, + "type": { + "the_type": { + "z": "Num" + }, + "type_id": 9 + } + } + } + } + ] + }, + "type": { + "the_type": { + "r": { + "z": "Num" + }, + "t": "Num", + "z": "Str" + }, + "type_id": 6 + } + } + } + } + ] + } + } + } +] \ No newline at end of file From 9a3fa81d6b65b9b0acd54413b8bd92251c1bc424 Mon Sep 17 00:00:00 2001 From: EvgSkv <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:39:53 -0700 Subject: [PATCH 038/141] Type inference: propositional predicates. --- type_inference/research/infer.py | 8 ++ .../research/integration_tests/run_tests.py | 1 + .../typing_aggregation_test.l | 17 +++ .../typing_aggregation_test.txt | 127 ++++++++++++++++++ type_inference/research/types_of_builtins.py | 8 +- 5 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 type_inference/research/integration_tests/typing_aggregation_test.l create mode 100644 type_inference/research/integration_tests/typing_aggregation_test.txt diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index c3712756..1a8145a0 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -116,6 +116,14 @@ def ActMindingBuiltinFieldTypes(self, node): fv['value']['expression']['type']['the_type'] = algebra.Intersect( fv['value']['expression']['type']['the_type'], self.types_of_builtins[p][fv['field']]) + if 'predicate' in node: + p = node['predicate']['predicate_name'] + if p in self.types_of_builtins: + for fv in node['predicate']['record']['field_value']: + if fv['field'] in self.types_of_builtins[p]: + fv['value']['expression']['type']['the_type'] = algebra.Intersect( + fv['value']['expression']['type']['the_type'], + self.types_of_builtins[p][fv['field']]) def MindBuiltinFieldTypes(self): Walk(self.rule, self.ActMindingBuiltinFieldTypes) diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index e805b84b..69e52456 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -27,6 +27,7 @@ def RunTypesTest(name, src=None, golden=None): def RunAll(): RunTypesTest('typing_basic_test') + RunTypesTest('typing_aggregation_test') if __name__ == '__main__': RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_aggregation_test.l b/type_inference/research/integration_tests/typing_aggregation_test.l new file mode 100644 index 00000000..10bd257c --- /dev/null +++ b/type_inference/research/integration_tests/typing_aggregation_test.l @@ -0,0 +1,17 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test(a:, b? += t) distinct :- T(a, t), Str(a); \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_aggregation_test.txt b/type_inference/research/integration_tests/typing_aggregation_test.txt new file mode 100644 index 00000000..cabdc452 --- /dev/null +++ b/type_inference/research/integration_tests/typing_aggregation_test.txt @@ -0,0 +1,127 @@ +[ + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "T", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "a" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "t" + } + } + } + } + ] + } + } + }, + { + "predicate": { + "predicate_name": "Str", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "a" + } + } + } + } + ] + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "Test(a:, b? += t) distinct :- T(a, t), Str(a)", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": "a", + "value": { + "expression": { + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "a" + } + } + } + }, + { + "field": "b", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "Agg+", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "t" + } + } + } + } + ] + } + }, + "type": { + "the_type": "Num", + "type_id": 1 + } + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 2d78deef..23a8450c 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -22,11 +22,15 @@ def TypesOfBultins(): 'logica_value': 'Num' }, 'Num': { - 'col0': 'Num', + 0: 'Num', 'logica_value': 'Num' }, 'Str': { - 'col0': 'Str', + 0: 'Str', 'logica_value': 'Str' + }, + 'Agg+': { + 0: 'Num', + 'logica_value': 'Num' } } \ No newline at end of file From 93bd2a9a4e5945c4e47770a2df29a7bde0f38cc6 Mon Sep 17 00:00:00 2001 From: EvgSkv <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 5 Jul 2023 23:53:32 -0700 Subject: [PATCH 039/141] Type inference: working with lists. --- type_inference/research/infer.py | 36 +++ .../research/integration_tests/run_tests.py | 1 + .../integration_tests/typing_lists_test.l | 17 ++ .../integration_tests/typing_lists_test.txt | 232 ++++++++++++++++++ 4 files changed, 286 insertions(+) create mode 100644 type_inference/research/integration_tests/typing_lists_test.l create mode 100644 type_inference/research/integration_tests/typing_lists_test.txt diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 1a8145a0..388d385f 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -31,6 +31,12 @@ def ExpressionsIterator(node): yield node[f] if 'record' in node and 'field_value' not in node['record']: yield node['record'] + if 'the_list' in node: + for e in node['the_list']['element']: + yield e + if 'inclusion' in node: + yield node['inclusion']['element'] + yield node['inclusion']['list'] def Walk(node, act): @@ -169,9 +175,39 @@ def ActMindingRecordLiterals(self, node): self.inference_complete = False fv['value']['expression']['type']['the_type'] = new_field_type + def ActMindingListLiterals(self, node): + if 'type' in node and 'literal' in node and 'the_list' in node['literal']: + list_type = node['type']['the_type'] + for e in node['literal']['the_list']['element']: + list_type, e_type = algebra.IntersectListElement( + list_type, e['type']['the_type']) + if e_type != e['type']['the_type']: + self.inference_complete = False + e['type']['the_type'] = e_type + + if list_type != node['type']['the_type']: + self.inference_complete = False + node['type']['the_type'] = list_type + + def ActMindingInclusion(self, node): + if 'inclusion' in node: + list_type = node['inclusion']['list']['type']['the_type'] + element_type = node['inclusion']['element']['type']['the_type'] + new_list_type, new_element_type = algebra.IntersectListElement( + list_type, element_type + ) + if list_type != new_list_type: + self.inference_complete = False + node['inclusion']['list']['type']['the_type'] = new_list_type + if element_type != new_element_type: + self.inference_complete = False + node['inclusion']['element']['type']['the_type'] = new_element_type + def IterateInference(self): while not self.inference_complete: self.inference_complete = True Walk(self.rule, self.ActUnifying) Walk(self.rule, self.ActUnderstandingSubscription) Walk(self.rule, self.ActMindingRecordLiterals) + Walk(self.rule, self.ActMindingListLiterals) + Walk(self.rule, self.ActMindingInclusion) diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index 69e52456..32984872 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -28,6 +28,7 @@ def RunTypesTest(name, src=None, golden=None): def RunAll(): RunTypesTest('typing_basic_test') RunTypesTest('typing_aggregation_test') + RunTypesTest('typing_lists_test') if __name__ == '__main__': RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_lists_test.l b/type_inference/research/integration_tests/typing_lists_test.l new file mode 100644 index 00000000..04593d85 --- /dev/null +++ b/type_inference/research/integration_tests/typing_lists_test.l @@ -0,0 +1,17 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test([1], a, b, c) :- a in [1, 2], T(b), a in b, c == ["hello"]; \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_lists_test.txt b/type_inference/research/integration_tests/typing_lists_test.txt new file mode 100644 index 00000000..2677de76 --- /dev/null +++ b/type_inference/research/integration_tests/typing_lists_test.txt @@ -0,0 +1,232 @@ +[ + { + "body": { + "conjunction": { + "conjunct": [ + { + "inclusion": { + "element": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "a" + } + }, + "list": { + "literal": { + "the_list": { + "element": [ + { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + }, + { + "literal": { + "the_number": { + "number": "2" + } + }, + "type": { + "the_type": "Num", + "type_id": 8 + } + } + ] + } + }, + "type": { + "the_type": [ + "Num" + ], + "type_id": 6 + } + } + } + }, + { + "predicate": { + "predicate_name": "T", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": [ + "Num" + ], + "type_id": 3 + }, + "variable": { + "var_name": "b" + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "a" + } + }, + "list": { + "type": { + "the_type": [ + "Num" + ], + "type_id": 3 + }, + "variable": { + "var_name": "b" + } + } + } + }, + { + "unification": { + "left_hand_side": { + "type": { + "the_type": [ + "Str" + ], + "type_id": 4 + }, + "variable": { + "var_name": "c" + } + }, + "right_hand_side": { + "literal": { + "the_list": { + "element": [ + { + "literal": { + "the_string": { + "the_string": "hello" + } + }, + "type": { + "the_type": "Str", + "type_id": 14 + } + } + ] + } + }, + "type": { + "the_type": [ + "Str" + ], + "type_id": 13 + } + } + } + } + ] + } + }, + "full_text": "Test([1], a, b, c) :- a in [1, 2], T(b), a in b, c == [\"hello\"]", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "literal": { + "the_list": { + "element": [ + { + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 1 + } + } + ] + } + }, + "type": { + "the_type": [ + "Num" + ], + "type_id": 0 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "a" + } + } + } + }, + { + "field": 2, + "value": { + "expression": { + "type": { + "the_type": [ + "Num" + ], + "type_id": 3 + }, + "variable": { + "var_name": "b" + } + } + } + }, + { + "field": 3, + "value": { + "expression": { + "type": { + "the_type": [ + "Str" + ], + "type_id": 4 + }, + "variable": { + "var_name": "c" + } + } + } + } + ] + } + } + } +] \ No newline at end of file From 82ac6a2d531e7c73ae32af58596adf537f2c6525 Mon Sep 17 00:00:00 2001 From: EvgSkv <71790359+EvgSkv@users.noreply.github.com> Date: Thu, 6 Jul 2023 00:03:33 -0700 Subject: [PATCH 040/141] Type inference: working with lists. --- .../integration_tests/typing_basic_test.txt | 20 +++++++++++++++---- type_inference/research/types_of_builtins.py | 4 ++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index 46c0389c..621f3bf5 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -101,7 +101,9 @@ "value": { "expression": { "type": { - "the_type": "Any", + "the_type": [ + "Any" + ], "type_id": 17 }, "variable": { @@ -117,11 +119,21 @@ { "inclusion": { "element": { + "type": { + "the_type": "Any", + "type_id": 18 + }, "variable": { "var_name": "e" } }, "list": { + "type": { + "the_type": [ + "Any" + ], + "type_id": 17 + }, "variable": { "var_name": "l" } @@ -159,7 +171,7 @@ "the_type": { "b": "Num" }, - "type_id": 20 + "type_id": 22 } }, "subscript": { @@ -172,7 +184,7 @@ }, "type": { "the_type": "Num", - "type_id": 18 + "type_id": 20 } }, "right_hand_side": { @@ -183,7 +195,7 @@ }, "type": { "the_type": "Num", - "type_id": 19 + "type_id": 21 } } } diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 23a8450c..5e2b7dc6 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -32,5 +32,9 @@ def TypesOfBultins(): 'Agg+': { 0: 'Num', 'logica_value': 'Num' + }, + 'List': { + 0: 'Any', + 'logica_value': ['Any'] } } \ No newline at end of file From c04794e1376f19fe30db18d5c8aad7dac3eafb18 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 9 Jul 2023 18:13:28 -0700 Subject: [PATCH 041/141] Type inference research: reference algebra. --- .gitignore | 2 +- type_inference/research/reference_algebra.py | 194 +++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 type_inference/research/reference_algebra.py diff --git a/.gitignore b/.gitignore index 84ada139..04f18983 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -docs/.DS_Store +*.DS_Store *.pyc diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py new file mode 100644 index 00000000..b5a87a0c --- /dev/null +++ b/type_inference/research/reference_algebra.py @@ -0,0 +1,194 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Type algebra working off of type references. + +class OpenRecord(dict): + def __str__(self): + return '{%s, ...}' % str(dict(self))[1:-1] + def __repr__(self): + return str(self) + + +class ClosedRecord(dict): + def __str__(self): + return str(dict(self)) + def __repr__(self): + return str(self) + + +class BadType(tuple): + def __str__(self): + return f'({self[0]} is incompatible with {self[1]})' + def __repr__(self): + return str(self) + + +class TypeReference: + def __init__(self, target): + self.target = target + + def Target(self): + result = self + while isinstance(result, TypeReference): + result = result.target + return result + + def TargetTypeClassName(self): + target = self.Target() + return type(target).__name__ + + @classmethod + def To(cls, target): + if isinstance(target, TypeReference): + return target + return TypeReference(target) + + def IsBadType(self): + return isinstance(self.Target(), BadType) + + +def ConcreteType(t): + if isinstance(t, TypeReference): + return t.Target() + assert (isinstance(t, BadType) or + isinstance(t, list) or + isinstance(t, dict) or + isinstance(t, str)) + return t + + +def Rank(x): + """Rank of the type, arbitrary order for sorting.""" + x = ConcreteType(x) + if isinstance(x, BadType): # Tuple means error. + return -1 + if x == 'Any': + return 0 + if x == 'Num': + return 1 + if x == 'Str': + return 2 + if isinstance(x, list): + return 3 + if isinstance(x, OpenRecord): + return 4 + if isinstance(x, ClosedRecord): + return 5 + assert False, 'Bad type: %s' % x + + +def Incompatible(a, b): + return BadType((a, b)) + + +def Unify(a, b): + """Unifies type reference a with type reference b.""" + assert isinstance(a, TypeReference) + assert isinstance(b, TypeReference) + concrete_a = ConcreteType(a) + concrete_b = ConcreteType(b) + + if isinstance(concrete_a, BadType) or isinstance(concrete_b, BadType): + return # Do nothing. + + if Rank(concrete_a) > Rank(concrete_b): + a, b = b, a + concrete_a, concrete_b = concrete_b, concrete_a + + if concrete_a == 'Any': + a.target = b + return + + if a in ('Num', 'Str'): + if a == b: + return # It's all fine. + a.target = Incompatible(a, b) # Type error: a is incompatible with b. + b.target = Incompatible(a, b) + + if isinstance(a, list): + if isinstance(b, list): + a_element, b_element = a + b + a_element = TypeReference.To(a_element) + b_element = TypeReference.To(b_element) + Unify(a_element, b_element) + # TODO: Make this correct. + if a_element.TargetTypeClassName() == 'BadType': + a.target = BadType(a, b) + b.target = BadType(a, b) + return + a.target = [a_element] + b.target = [b_element] + return + a.target = BadType(a, b) + b.target = BadType(b, a) + return + + if isinstance(concrete_a, OpenRecord): + if isinstance(concrete_b, OpenRecord): + UnifyFriendlyRecords(a, b, OpenRecord) + return + if isinstance(concrete_b, ClosedRecord): + if set(concrete_a) <= set(concrete_b): + UnifyFriendlyRecords(a, b, ClosedRecord) + return + a.target = Incompatible(a, b) + b.target = Incompatible(b, a) + assert False + + if isinstance(a, ClosedRecord): + if isinstance(b, ClosedRecord): + if set(a) == set(b): + return UnifyFriendlyRecords(a, b, ClosedRecord) + a.target = Incompatible(a, b) + b.target = Incompatible(b, a) + assert False + assert False + + +def UnifyFriendlyRecords(a, b, record_type): + """Intersecting records assuming that their fields are compatible.""" + concrete_a = ConcreteType(a) + concrete_b = ConcreteType(b) + result = {} + for f in set(concrete_a) | set(concrete_b): + x = TypeReference.To('Any') + if f in concrete_a: + Unify(x, TypeReference.To(concrete_a[f])) + if f in concrete_b: + Unify(x, TypeReference.To(concrete_b[f])) + if x.TargetTypeClassName() == 'BadType': # Ooops, field type error. + a.target = Incompatible(a, b) + b.target = Incompatible(b, a) + result[f] = x + a.target = record_type(result) + b.target = record_type(result) + + +def UnifyListElement(a_list, b_element): + """Analysis of expression `b in a`.""" + b = TypeReference([b_element]) + Unify(a_list, b) + + # TODO: Prohibit lists of lists. + + +def IntersectRecordField(a_record, field_name, b_field_value): + """Analysis of expresson `a.f = b`.""" + b = TypeReference(OpenRecord({field_name: b_field_value})) + Unify(a_record, b) + + From 493c69c669dc060e9b6b8df05786801dc4ad7d7f Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 9 Jul 2023 20:34:04 -0700 Subject: [PATCH 042/141] Type inference research: reference algebra. --- type_inference/research/reference_algebra.py | 93 ++++++++++++++++---- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index b5a87a0c..a789b551 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Type algebra working off of type references. class OpenRecord(dict): def __str__(self): @@ -36,16 +35,18 @@ def __str__(self): def __repr__(self): return str(self) - class TypeReference: def __init__(self, target): self.target = target + def WeMustGoDeeper(self): + return isinstance(self.target, TypeReference) + def Target(self): result = self - while isinstance(result, TypeReference): + while result.WeMustGoDeeper(): result = result.target - return result + return result.target def TargetTypeClassName(self): target = self.Target() @@ -59,6 +60,12 @@ def To(cls, target): def IsBadType(self): return isinstance(self.Target(), BadType) + + def __str__(self): + return str(self.target) + '@' + hex(id(self)) + + def __repr__(self): + return str(self) def ConcreteType(t): @@ -70,6 +77,19 @@ def ConcreteType(t): isinstance(t, str)) return t +def VeryConcreteType(t): + c = ConcreteType(t) + if isinstance(c, BadType) or isinstance(c, str): + return c + + if isinstance(c, list): + return [VeryConcreteType(e) for e in c] + + if isinstance(c, dict): + return type(c)({f: VeryConcreteType(v) for f, v in c.items()}) + + assert False + def Rank(x): """Rank of the type, arbitrary order for sorting.""" @@ -97,6 +117,18 @@ def Incompatible(a, b): def Unify(a, b): """Unifies type reference a with type reference b.""" + original_a = a + original_b = b + while a.WeMustGoDeeper(): + a = a.target + while b.WeMustGoDeeper(): + b = b.target + if original_a != a: + original_a.target = a + if original_b != b: + original_b.target = b + if id(a) == id(b): + return assert isinstance(a, TypeReference) assert isinstance(b, TypeReference) concrete_a = ConcreteType(a) @@ -113,15 +145,18 @@ def Unify(a, b): a.target = b return - if a in ('Num', 'Str'): - if a == b: + if concrete_a in ('Num', 'Str'): + if concrete_a == concrete_b: return # It's all fine. - a.target = Incompatible(a, b) # Type error: a is incompatible with b. - b.target = Incompatible(a, b) + # Type error: a is incompatible with b. + a.target, b.target = ( + Incompatible(a.target, b.target), + Incompatible(b.target, a.target)) + return - if isinstance(a, list): - if isinstance(b, list): - a_element, b_element = a + b + if isinstance(concrete_a, list): + if isinstance(concrete_b, list): + a_element, b_element = concrete_a + concrete_b a_element = TypeReference.To(a_element) b_element = TypeReference.To(b_element) Unify(a_element, b_element) @@ -152,9 +187,11 @@ def Unify(a, b): if isinstance(a, ClosedRecord): if isinstance(b, ClosedRecord): if set(a) == set(b): - return UnifyFriendlyRecords(a, b, ClosedRecord) + UnifyFriendlyRecords(a, b, ClosedRecord) + return a.target = Incompatible(a, b) b.target = Incompatible(b, a) + return assert False assert False @@ -174,8 +211,8 @@ def UnifyFriendlyRecords(a, b, record_type): a.target = Incompatible(a, b) b.target = Incompatible(b, a) result[f] = x - a.target = record_type(result) - b.target = record_type(result) + a.target = TypeReference(record_type(result)) + b.target = a.target def UnifyListElement(a_list, b_element): @@ -186,9 +223,33 @@ def UnifyListElement(a_list, b_element): # TODO: Prohibit lists of lists. -def IntersectRecordField(a_record, field_name, b_field_value): +def UnifyRecordField(a_record, field_name, b_field_value): """Analysis of expresson `a.f = b`.""" b = TypeReference(OpenRecord({field_name: b_field_value})) Unify(a_record, b) - +class TypeStructureCopier: + def __init__(self): + self.id_to_reference = {} + + def CopyConcreteOrReferenceType(self, t): + if isinstance(t, TypeReference): + return self.CopyTypeReference(t) + return self.CopyConcreteType(t) + + def CopyConcreteType(self, t): + if isinstance(t, str): + return t + if isinstance(t, list): + return [self.CopyConcreteOrReferenceType(e) for e in t] + if isinstance(t, dict): + c = type(t) + return c({k: self.CopyConcreteOrReferenceType(v) for k, v in t.items()}) + assert False, (t, type(t)) + + def CopyTypeReference(self, t): + if id(t) not in self.id_to_reference: + target = self.CopyConcreteOrReferenceType(t.target) + n = TypeReference(target) + self.id_to_reference[id(t)] = n + return self.id_to_reference[id(t)] From 4d5071cb9721020cfad728ad52fd7d20ad63e1a9 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:29:21 -0700 Subject: [PATCH 043/141] Optimizing type inference. --- type_inference/research/reference_algebra.py | 61 +++++++++++++++----- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index a789b551..0ad3ef4c 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -14,6 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +if '.' not in __package__: + from common import color +else: + try: + from ...common import color + except: + from common import color + class OpenRecord(dict): def __str__(self): @@ -31,7 +39,12 @@ def __repr__(self): class BadType(tuple): def __str__(self): - return f'({self[0]} is incompatible with {self[1]})' + colored_t1 = color.Format('{warning}{t}{end}', args_dict={'t': RenderType(self[0])}) + colored_t2 = color.Format('{warning}{t}{end}', args_dict={'t': RenderType(self[1])}) + + return ( + f'is implied to be {colored_t1} and ' + + f'simultaneously {colored_t2}, which is impossible.') def __repr__(self): return str(self) @@ -66,7 +79,22 @@ def __str__(self): def __repr__(self): return str(self) - + + def CloseRecord(self): + a = self + while a.WeMustGoDeeper(): + a = a.target + assert isinstance(a.target, dict) + a.target = ClosedRecord(a.target) + +def RenderType(t): + if isinstance(t, str): + return t + if isinstance(t, list): + return '[%s]' % RenderType(t[0]) + if isinstance(t, dict): + return '{%s}' % ', '.join('%s: %s' % (k, RenderType(v)) + for k, v in t.items()) def ConcreteType(t): if isinstance(t, TypeReference): @@ -79,7 +107,9 @@ def ConcreteType(t): def VeryConcreteType(t): c = ConcreteType(t) - if isinstance(c, BadType) or isinstance(c, str): + if isinstance(c, BadType): + return BadType(VeryConcreteType(e) for e in c) + if isinstance(c, str): return c if isinstance(c, list): @@ -162,14 +192,17 @@ def Unify(a, b): Unify(a_element, b_element) # TODO: Make this correct. if a_element.TargetTypeClassName() == 'BadType': - a.target = BadType(a, b) - b.target = BadType(a, b) + a.target, b.target = ( + Incompatible(a.target, b.target), + Incompatible(b.target, a.target)) return a.target = [a_element] b.target = [b_element] return - a.target = BadType(a, b) - b.target = BadType(b, a) + a.target, b.target = ( + Incompatible(a.target, b.target), + Incompatible(b.target, a.target)) + return if isinstance(concrete_a, OpenRecord): @@ -180,20 +213,22 @@ def Unify(a, b): if set(concrete_a) <= set(concrete_b): UnifyFriendlyRecords(a, b, ClosedRecord) return - a.target = Incompatible(a, b) - b.target = Incompatible(b, a) + a.target, b.target = ( + Incompatible(a.target, b.target), + Incompatible(b.target, a.target)) + return assert False - if isinstance(a, ClosedRecord): - if isinstance(b, ClosedRecord): - if set(a) == set(b): + if isinstance(concrete_a, ClosedRecord): + if isinstance(concrete_b, ClosedRecord): + if set(concrete_a) == set(concrete_b): UnifyFriendlyRecords(a, b, ClosedRecord) return a.target = Incompatible(a, b) b.target = Incompatible(b, a) return assert False - assert False + assert False, (a, type(a)) def UnifyFriendlyRecords(a, b, record_type): From a4cda703c9da3cc1a4094bd13a38b31e22232626 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:29:29 -0700 Subject: [PATCH 044/141] Optimizing type inference. --- logica.py | 1 + parser_py/parse.py | 13 + type_inference/research/infer.py | 180 ++++++--- .../research/integration_tests/run_tests.py | 1 + .../typing_aggregation_test.txt | 5 + .../integration_tests/typing_basic_test.txt | 24 ++ .../integration_tests/typing_lists_test.txt | 15 + .../integration_tests/typing_nested_test.l | 21 + .../integration_tests/typing_nested_test.txt | 365 ++++++++++++++++++ type_inference/research/types_of_builtins.py | 52 ++- 10 files changed, 617 insertions(+), 60 deletions(-) create mode 100644 type_inference/research/integration_tests/typing_nested_test.l create mode 100644 type_inference/research/integration_tests/typing_nested_test.txt diff --git a/logica.py b/logica.py index c14ff6fc..151e8027 100755 --- a/logica.py +++ b/logica.py @@ -174,6 +174,7 @@ def main(argv): if command == 'infer_types': typing_engine = infer.TypesInferenceEngine(parsed_rules) typing_engine.InferTypes() + # print(parsed_rules) print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 diff --git a/parser_py/parse.py b/parser_py/parse.py index 027496f3..0ff92298 100755 --- a/parser_py/parse.py +++ b/parser_py/parse.py @@ -83,6 +83,14 @@ def Pieces(self): self.heritage[self.start:self.stop], self.heritage[self.stop:]) + def Display(self): + a, b, c = self.Pieces() + if not b: + b = '' + return color.Format( + '{before}{warning}{error_text}{end}{after}', + dict(before=a, error_text=b, after=c)) + class ParsingException(Exception): """Exception thrown by parsing.""" @@ -716,6 +724,11 @@ def ParseImplication(s): def ParseExpression(s): + e = ActuallyParseExpression(s) + e['expression_heritage'] = s + return e + +def ActuallyParseExpression(s): """Parsing logica.Expression.""" v = ParseCombine(s) if v: diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 388d385f..39985731 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -15,11 +15,77 @@ # limitations under the License. if '.' not in __package__: - from type_inference.research import algebra + from type_inference.research import reference_algebra from type_inference.research import types_of_builtins + from common import color else: - from ..research import algebra + from ..research import reference_algebra from ..research import types_of_builtins + try: + from ...common import color + except: + from common import color + + +class ContextualizedError: + def __init__(self): + self.type_error = None + self.context_string = None + self.refers_to_variable = None + self.refers_to_expression = None + + def Replace(self, type_error, context_string, refers_to_variable, refers_to_expression): + self.type_error = type_error + self.context_string = context_string + self.refers_to_variable = refers_to_variable + self.refers_to_expression = refers_to_expression + + def ReplaceIfMoreUseful(self, type_error, context_string, refers_to_variable, refers_to_expression): + if self.type_error is None: + self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) + elif self.refers_to_variable is None and refers_to_variable: + self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) + elif 'literal' in self.refers_to_expression: + self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) + + else: + pass + + def NiceMessage(self): + result_lines = [ + color.Format('{underline}Type analysis:{end}'), + self.context_string, '', + color.Format('[ {error}Error{end} ] ') + self.HelpfulErrorMessage()] + + return '\n'.join(result_lines) + + def HelpfulErrorMessage(self): + result = str(self.type_error) + + if (isinstance(self.type_error[0], dict) and + isinstance(self.type_error[1], dict)): + if isinstance(self.type_error[0], + reference_algebra.ClosedRecord): + a, b = self.type_error + else: + b, a = self.type_error + if (isinstance(a, reference_algebra.ClosedRecord) and + isinstance(b, reference_algebra.OpenRecord) and + list(b)[0] not in a.keys() ): + result = ( + 'record ' + str(a) + ' does not have field ' + list(b)[0] + '.' + ) + + if self.refers_to_variable: + result = color.Format( + 'Variable {warning}%s{end} ' % + self.refers_to_variable) + result + else: + result = color.Format( + 'Expression {warning}%s{end} ' % + str(self.refers_to_expression['expression_heritage']) + result + ) + return result def ExpressionFields(): @@ -55,9 +121,9 @@ def ActMindingPodLiterals(node): for e in ExpressionsIterator(node): if 'literal' in e: if 'the_number' in e['literal']: - e['type']['the_type'] = algebra.Intersect(e['type']['the_type'], 'Num') + reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Num')) if 'the_string' in e['literal']: - e['type']['the_type'] = algebra.Intersect(e['type']['the_type'], 'Str') + reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Str')) class TypesInferenceEngine: @@ -69,15 +135,23 @@ def InferTypes(self): for rule in self.parsed_rules: t = TypeInferenceForRule(rule) t.PerformInference() + + def Concretize(node): + if isinstance(node, dict): + if 'type' in node: + node['type']['the_type'] = reference_algebra.VeryConcreteType( + node['type']['the_type']) + for rule in self.parsed_rules: + Walk(rule, Concretize) class TypeInferenceForRule: def __init__(self, rule): self.rule = rule - self.inference_complete = False self.variable_type = {} self.type_id_counter = 0 self.types_of_builtins = types_of_builtins.TypesOfBultins() + self.found_error = None def PerformInference(self): self.InitTypes() @@ -97,10 +171,10 @@ def ActInitializingTypes(self, node): var_name = e['variable']['var_name'] use_type = self.variable_type.get( var_name, - {'the_type': 'Any', 'type_id': i}) + {'the_type': reference_algebra.TypeReference('Any'), 'type_id': i}) self.variable_type[var_name] = use_type else: - use_type = {'the_type': 'Any', 'type_id': i} + use_type = {'the_type': reference_algebra.TypeReference('Any'), 'type_id': i} e['type'] = use_type def InitTypes(self): @@ -114,22 +188,27 @@ def ActMindingBuiltinFieldTypes(self, node): if 'call' in e: p = e['call']['predicate_name'] if p in self.types_of_builtins: - e['type']['the_type'] = algebra.Intersect( + copier = reference_algebra.TypeStructureCopier() + copy = copier.CopyConcreteOrReferenceType + + reference_algebra.Unify( e['type']['the_type'], - self.types_of_builtins[p]['logica_value']) + copy(self.types_of_builtins[p]['logica_value'])) for fv in e['call']['record']['field_value']: if fv['field'] in self.types_of_builtins[p]: - fv['value']['expression']['type']['the_type'] = algebra.Intersect( + reference_algebra.Unify( fv['value']['expression']['type']['the_type'], - self.types_of_builtins[p][fv['field']]) + copy(self.types_of_builtins[p][fv['field']])) if 'predicate' in node: p = node['predicate']['predicate_name'] if p in self.types_of_builtins: + copier = reference_algebra.TypeStructureCopier() + copy = copier.CopyConcreteOrReferenceType for fv in node['predicate']['record']['field_value']: if fv['field'] in self.types_of_builtins[p]: - fv['value']['expression']['type']['the_type'] = algebra.Intersect( + reference_algebra.Unify( fv['value']['expression']['type']['the_type'], - self.types_of_builtins[p][fv['field']]) + copy(self.types_of_builtins[p][fv['field']])) def MindBuiltinFieldTypes(self): Walk(self.rule, self.ActMindingBuiltinFieldTypes) @@ -138,27 +217,15 @@ def ActUnifying(self, node): if 'unification' in node: left_type = node['unification']['left_hand_side']['type']['the_type'] right_type = node['unification']['right_hand_side']['type']['the_type'] - new_type = algebra.Intersect(left_type, right_type) - if new_type != left_type: - self.inference_complete = False - node['unification']['left_hand_side']['type']['the_type'] = new_type - if new_type != right_type: - self.inference_complete = False - node['unification']['right_hand_side']['type']['the_type'] = new_type + reference_algebra.Unify(left_type, right_type) def ActUnderstandingSubscription(self, node): if 'subscript' in node and 'record' in node['subscript']: record_type = node['subscript']['record']['type']['the_type'] field_type = node['type']['the_type'] field_name = node['subscript']['subscript']['literal']['the_symbol']['symbol'] - new_record_type, new_field_type = algebra.IntersectRecordField( + reference_algebra.UnifyRecordField( record_type, field_name, field_type) - if new_record_type != record_type: - self.inference_complete = False - node['subscript']['record']['type']['the_type'] = new_record_type - if new_field_type != field_type: - self.inference_complete = False - node['type']['the_type'] = new_field_type def ActMindingRecordLiterals(self, node): if 'type' in node and 'record' in node: @@ -166,48 +233,49 @@ def ActMindingRecordLiterals(self, node): record_type = node['type']['the_type'] field_type = fv['value']['expression']['type']['the_type'] field_name = fv['field'] - new_record_type, new_field_type = algebra.IntersectRecordField( + reference_algebra.UnifyRecordField( record_type, field_name, field_type) - if new_record_type != record_type: - self.inference_complete = False - node['type']['the_type'] = new_record_type - if new_field_type != field_type: - self.inference_complete = False - fv['value']['expression']['type']['the_type'] = new_field_type + node['type']['the_type'].CloseRecord() def ActMindingListLiterals(self, node): if 'type' in node and 'literal' in node and 'the_list' in node['literal']: list_type = node['type']['the_type'] for e in node['literal']['the_list']['element']: - list_type, e_type = algebra.IntersectListElement( + reference_algebra.UnifyListElement( list_type, e['type']['the_type']) - if e_type != e['type']['the_type']: - self.inference_complete = False - e['type']['the_type'] = e_type - - if list_type != node['type']['the_type']: - self.inference_complete = False - node['type']['the_type'] = list_type def ActMindingInclusion(self, node): if 'inclusion' in node: list_type = node['inclusion']['list']['type']['the_type'] element_type = node['inclusion']['element']['type']['the_type'] - new_list_type, new_element_type = algebra.IntersectListElement( + reference_algebra.UnifyListElement( list_type, element_type ) - if list_type != new_list_type: - self.inference_complete = False - node['inclusion']['list']['type']['the_type'] = new_list_type - if element_type != new_element_type: - self.inference_complete = False - node['inclusion']['element']['type']['the_type'] = new_element_type def IterateInference(self): - while not self.inference_complete: - self.inference_complete = True - Walk(self.rule, self.ActUnifying) - Walk(self.rule, self.ActUnderstandingSubscription) - Walk(self.rule, self.ActMindingRecordLiterals) - Walk(self.rule, self.ActMindingListLiterals) - Walk(self.rule, self.ActMindingInclusion) + Walk(self.rule, self.ActMindingRecordLiterals) + Walk(self.rule, self.ActUnifying) + Walk(self.rule, self.ActUnderstandingSubscription) + Walk(self.rule, self.ActMindingListLiterals) + Walk(self.rule, self.ActMindingInclusion) + + self.found_error = self.SearchTypeErrors() + if self.found_error.type_error: + print(self.found_error.NiceMessage()) + + def SearchTypeErrors(self): + found_error = ContextualizedError() + def LookForError(node): + nonlocal found_error + if 'type' in node: + t = reference_algebra.VeryConcreteType(node['type']['the_type']) + if isinstance(t, reference_algebra.BadType): + if 'variable' in node: + v = node['variable']['var_name'] + else: + v = None + found_error.ReplaceIfMoreUseful( + t, node['expression_heritage'].Display(), v, + node) + Walk(self.rule, LookForError) + return found_error diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index 32984872..3ead62fe 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -29,6 +29,7 @@ def RunAll(): RunTypesTest('typing_basic_test') RunTypesTest('typing_aggregation_test') RunTypesTest('typing_lists_test') + RunTypesTest('typing_nested_test') if __name__ == '__main__': RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_aggregation_test.txt b/type_inference/research/integration_tests/typing_aggregation_test.txt index cabdc452..564ec8b9 100644 --- a/type_inference/research/integration_tests/typing_aggregation_test.txt +++ b/type_inference/research/integration_tests/typing_aggregation_test.txt @@ -12,6 +12,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "a", "type": { "the_type": "Str", "type_id": 0 @@ -26,6 +27,7 @@ "field": 1, "value": { "expression": { + "expression_heritage": "t", "type": { "the_type": "Num", "type_id": 2 @@ -49,6 +51,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "a", "type": { "the_type": "Str", "type_id": 0 @@ -76,6 +79,7 @@ "field": "a", "value": { "expression": { + "expression_heritage": "a", "type": { "the_type": "Str", "type_id": 0 @@ -99,6 +103,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "t", "type": { "the_type": "Num", "type_id": 2 diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index 621f3bf5..3e2d6737 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -6,6 +6,7 @@ { "unification": { "left_hand_side": { + "expression_heritage": "x", "type": { "the_type": "Num", "type_id": 2 @@ -23,6 +24,7 @@ "field": "left", "value": { "expression": { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" @@ -39,6 +41,7 @@ "field": "right", "value": { "expression": { + "expression_heritage": "z", "type": { "the_type": "Num", "type_id": 3 @@ -52,6 +55,7 @@ ] } }, + "expression_heritage": "1 + z", "type": { "the_type": "Num", "type_id": 12 @@ -68,6 +72,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "z", "type": { "the_type": "Num", "type_id": 3 @@ -82,6 +87,7 @@ "field": 1, "value": { "expression": { + "expression_heritage": "w", "type": { "the_type": { "a": { @@ -100,6 +106,7 @@ "field": 2, "value": { "expression": { + "expression_heritage": "l", "type": { "the_type": [ "Any" @@ -119,6 +126,7 @@ { "inclusion": { "element": { + "expression_heritage": "e", "type": { "the_type": "Any", "type_id": 18 @@ -128,6 +136,7 @@ } }, "list": { + "expression_heritage": "l", "type": { "the_type": [ "Any" @@ -143,10 +152,13 @@ { "unification": { "left_hand_side": { + "expression_heritage": "w.a.b", "subscript": { "record": { + "expression_heritage": "w.a", "subscript": { "record": { + "expression_heritage": "w", "type": { "the_type": { "a": { @@ -188,6 +200,7 @@ } }, "right_hand_side": { + "expression_heritage": "7", "literal": { "the_number": { "number": "7" @@ -212,6 +225,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" @@ -228,6 +242,7 @@ "field": 1, "value": { "expression": { + "expression_heritage": "\"a\"", "literal": { "the_string": { "the_string": "a" @@ -244,6 +259,7 @@ "field": 2, "value": { "expression": { + "expression_heritage": "x", "type": { "the_type": "Num", "type_id": 2 @@ -258,6 +274,7 @@ "field": 3, "value": { "expression": { + "expression_heritage": "z", "type": { "the_type": "Num", "type_id": 3 @@ -272,8 +289,10 @@ "field": 4, "value": { "expression": { + "expression_heritage": "w.a", "subscript": { "record": { + "expression_heritage": "w", "type": { "the_type": { "a": { @@ -307,12 +326,14 @@ "field": 5, "value": { "expression": { + "expression_heritage": "{t: 1, z: \"a\", r: {z: 1}}", "record": { "field_value": [ { "field": "t", "value": { "expression": { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" @@ -329,6 +350,7 @@ "field": "z", "value": { "expression": { + "expression_heritage": "\"a\"", "literal": { "the_string": { "the_string": "a" @@ -345,12 +367,14 @@ "field": "r", "value": { "expression": { + "expression_heritage": "{z: 1}", "record": { "field_value": [ { "field": "z", "value": { "expression": { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" diff --git a/type_inference/research/integration_tests/typing_lists_test.txt b/type_inference/research/integration_tests/typing_lists_test.txt index 2677de76..349716ac 100644 --- a/type_inference/research/integration_tests/typing_lists_test.txt +++ b/type_inference/research/integration_tests/typing_lists_test.txt @@ -6,6 +6,7 @@ { "inclusion": { "element": { + "expression_heritage": "a", "type": { "the_type": "Num", "type_id": 2 @@ -15,10 +16,12 @@ } }, "list": { + "expression_heritage": "[1, 2]", "literal": { "the_list": { "element": [ { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" @@ -30,6 +33,7 @@ } }, { + "expression_heritage": "2", "literal": { "the_number": { "number": "2" @@ -61,6 +65,7 @@ "field": 0, "value": { "expression": { + "expression_heritage": "b", "type": { "the_type": [ "Num" @@ -80,6 +85,7 @@ { "inclusion": { "element": { + "expression_heritage": "a", "type": { "the_type": "Num", "type_id": 2 @@ -89,6 +95,7 @@ } }, "list": { + "expression_heritage": "b", "type": { "the_type": [ "Num" @@ -104,6 +111,7 @@ { "unification": { "left_hand_side": { + "expression_heritage": "c", "type": { "the_type": [ "Str" @@ -115,10 +123,12 @@ } }, "right_hand_side": { + "expression_heritage": "[\"hello\"]", "literal": { "the_list": { "element": [ { + "expression_heritage": "\"hello\"", "literal": { "the_string": { "the_string": "hello" @@ -153,10 +163,12 @@ "field": 0, "value": { "expression": { + "expression_heritage": "[1]", "literal": { "the_list": { "element": [ { + "expression_heritage": "1", "literal": { "the_number": { "number": "1" @@ -183,6 +195,7 @@ "field": 1, "value": { "expression": { + "expression_heritage": "a", "type": { "the_type": "Num", "type_id": 2 @@ -197,6 +210,7 @@ "field": 2, "value": { "expression": { + "expression_heritage": "b", "type": { "the_type": [ "Num" @@ -213,6 +227,7 @@ "field": 3, "value": { "expression": { + "expression_heritage": "c", "type": { "the_type": [ "Str" diff --git a/type_inference/research/integration_tests/typing_nested_test.l b/type_inference/research/integration_tests/typing_nested_test.l new file mode 100644 index 00000000..25691ac2 --- /dev/null +++ b/type_inference/research/integration_tests/typing_nested_test.l @@ -0,0 +1,21 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test(y) :- + l = [{a: {b: {c: [1]}}}], + x in l, + c = x.a.b.c, + y in c; \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_nested_test.txt b/type_inference/research/integration_tests/typing_nested_test.txt new file mode 100644 index 00000000..adc97031 --- /dev/null +++ b/type_inference/research/integration_tests/typing_nested_test.txt @@ -0,0 +1,365 @@ +[ + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "=", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "l", + "type": { + "the_type": [ + { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + } + ], + "type_id": 1 + }, + "variable": { + "var_name": "l" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "[{a: {b: {c: [1]}}}]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "{a: {b: {c: [1]}}}", + "record": { + "field_value": [ + { + "field": "a", + "value": { + "expression": { + "expression_heritage": "{b: {c: [1]}}", + "record": { + "field_value": [ + { + "field": "b", + "value": { + "expression": { + "expression_heritage": "{c: [1]}", + "record": { + "field_value": [ + { + "field": "c", + "value": { + "expression": { + "expression_heritage": "[1]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + ] + } + }, + "type": { + "the_type": [ + "Num" + ], + "type_id": 6 + } + } + } + } + ] + }, + "type": { + "the_type": { + "c": [ + "Num" + ] + }, + "type_id": 5 + } + } + } + } + ] + }, + "type": { + "the_type": { + "b": { + "c": [ + "Num" + ] + } + }, + "type_id": 4 + } + } + } + } + ] + }, + "type": { + "the_type": { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + }, + "type_id": 3 + } + } + ] + } + }, + "type": { + "the_type": [ + { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + } + ], + "type_id": 2 + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "expression_heritage": "x", + "type": { + "the_type": { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + }, + "type_id": 8 + }, + "variable": { + "var_name": "x" + } + }, + "list": { + "expression_heritage": "l", + "type": { + "the_type": [ + { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + } + ], + "type_id": 1 + }, + "variable": { + "var_name": "l" + } + } + } + }, + { + "predicate": { + "predicate_name": "=", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "c", + "type": { + "the_type": [ + "Num" + ], + "type_id": 10 + }, + "variable": { + "var_name": "c" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "x.a.b.c", + "subscript": { + "record": { + "expression_heritage": "x.a.b", + "subscript": { + "record": { + "expression_heritage": "x.a", + "subscript": { + "record": { + "expression_heritage": "x", + "type": { + "the_type": { + "a": { + "b": { + "c": [ + "Num" + ] + } + } + }, + "type_id": 8 + }, + "variable": { + "var_name": "x" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "a" + } + } + } + }, + "type": { + "the_type": { + "b": { + "c": [ + "Num" + ] + } + }, + "type_id": 13 + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "b" + } + } + } + }, + "type": { + "the_type": { + "c": [ + "Num" + ] + }, + "type_id": 12 + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "c" + } + } + } + }, + "type": { + "the_type": [ + "Num" + ], + "type_id": 11 + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "expression_heritage": "y", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "y" + } + }, + "list": { + "expression_heritage": "c", + "type": { + "the_type": [ + "Num" + ], + "type_id": 10 + }, + "variable": { + "var_name": "c" + } + } + } + } + ] + } + }, + "full_text": "Test(y) :-\n l = [{a: {b: {c: [1]}}}],\n x in l,\n c = x.a.b.c,\n y in c", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "y", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "y" + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 5e2b7dc6..52a63471 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -14,13 +14,43 @@ # See the License for the specific language governing permissions and # limitations under the License. +if '.' not in __package__: + from type_inference.research import reference_algebra +else: + from ..research import reference_algebra + def TypesOfBultins(): - return { + x = reference_algebra.TypeReference('Any') + y = reference_algebra.TypeReference('Any') + list_of_x = reference_algebra.TypeReference('Any') + reference_algebra.UnifyListElement(list_of_x, x) + + types_of_predicate = { + '=': { + 'left': x, + 'right': x, + 'logica_value': x + }, + '++': { + 'left': 'Str', + 'right': 'Str', + 'logica_value': 'Str' + }, '+': { 'left': 'Num', 'right': 'Num', 'logica_value': 'Num' }, + '*': { + 'left': 'Num', + 'right': 'Num', + 'logica_value': 'Num' + }, + '^': { + 'left': 'Num', + 'right': 'Num', + 'logica_value': 'Num' + }, 'Num': { 0: 'Num', 'logica_value': 'Num' @@ -34,7 +64,21 @@ def TypesOfBultins(): 'logica_value': 'Num' }, 'List': { - 0: 'Any', - 'logica_value': ['Any'] + 0: x, + 'logica_value': list_of_x + }, + '->': { + 'left': x, + 'right': y, + 'logica_value': reference_algebra.ClosedRecord({'arg': x, 'value': y}) + }, + 'ArgMin': { + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 'logica_value': x } - } \ No newline at end of file + } + return { + p: {k: reference_algebra.TypeReference(v) + for k, v in types.items()} + for p, types in types_of_predicate.items() + } From d715c8efdb71be141021ff0cb8a700dc4edf1d4f Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 12 Jul 2023 14:14:57 -0700 Subject: [PATCH 045/141] Type inference research: multi-rules predicates. --- logica.py | 15 +- type_inference/research/infer.py | 164 ++++++++++++++++--- type_inference/research/reference_algebra.py | 2 + type_inference/research/types_of_builtins.py | 4 + 4 files changed, 157 insertions(+), 28 deletions(-) diff --git a/logica.py b/logica.py index 151e8027..5631f71f 100755 --- a/logica.py +++ b/logica.py @@ -132,7 +132,7 @@ def main(argv): 'GoodIdea(snack: "carrots")\'') return 1 - if len(argv) == 3 and argv[2] in ['parse', 'infer_types']: + if len(argv) == 3 and argv[2] in ['parse', 'infer_types', 'show_signatures']: pass # compile needs just 2 actual arguments. else: if len(argv) < 4: @@ -147,7 +147,8 @@ def main(argv): command = argv[2] - commands = ['parse', 'print', 'run', 'run_to_csv', 'run_in_terminal', 'infer_types'] + commands = ['parse', 'print', 'run', 'run_to_csv', 'run_in_terminal', + 'infer_types', 'show_signatures'] if command not in commands: print(color.Format('Unknown command {warning}{command}{end}. ' @@ -178,6 +179,16 @@ def main(argv): print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) return 0 + if command == 'show_signatures': + typing_engine = infer.TypesInferenceEngine(parsed_rules) + typing_engine.InferTypes() + print(typing_engine.ShowPredicateTypes()) + type_error_checker = infer.TypeErrorChecker(parsed_rules) + type_error_checker.CheckForError() + # print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) + + return 0 + predicates = argv[3] user_flags = ReadUserFlags(parsed_rules, argv[4:]) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 39985731..2121226c 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -130,12 +130,41 @@ class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules self.predicate_argumets_types = {} - + self.dependencies = BuildDependencies(self.parsed_rules) + self.complexities = BuildComplexities(self.dependencies) + self.parsed_rules = list(sorted(self.parsed_rules, key=lambda x: self.complexities[x['head']['predicate_name']])) + self.types_of_builtins = types_of_builtins.TypesOfBultins() + + def UpdateTypes(self, rule): + predicate_name = rule['head']['predicate_name'] + if predicate_name in self.types_of_builtins: + predicate_signature = self.types_of_builtins[predicate_name] + else: + predicate_signature = {} + for fv in rule['head']['record']['field_value']: + field_name = fv['field'] + predicate_signature[field_name] = reference_algebra.TypeReference('Any') + self.types_of_builtins[predicate_name] = predicate_signature + + for fv in rule['head']['record']['field_value']: + field_name = fv['field'] + v = fv['value'] + if 'expression' in v: + value = v['expression'] + else: + value = v['aggregation']['expression'] + value_type = value['type']['the_type'] + reference_algebra.Unify( + predicate_signature[field_name], + value_type) + + def InferTypes(self): for rule in self.parsed_rules: - t = TypeInferenceForRule(rule) + t = TypeInferenceForRule(rule, self.types_of_builtins) t.PerformInference() - + self.UpdateTypes(rule) + def Concretize(node): if isinstance(node, dict): if 'type' in node: @@ -143,15 +172,49 @@ def Concretize(node): node['type']['the_type']) for rule in self.parsed_rules: Walk(rule, Concretize) + + def ShowPredicateTypes(self): + result_lines = [] + for predicate_name, signature in self.types_of_builtins.items(): + result_lines.append(RenderPredicateSignature(predicate_name, signature)) + return '\n'.join(result_lines) + + +def BuildDependencies(rules): + def ExtractDendencies(rule): + p = rule['head']['predicate_name'] + dependencies = [] + def ExtractPredicateName(node): + if 'predicate_name' in node: + dependencies.append(node['predicate_name']) + Walk(rule, ExtractPredicateName) + return p, dependencies + result = {} + for rule in rules: + p, ds = ExtractDendencies(rule) + result[p] = list(sorted(set(ds) - set([p]))) + return result + +def BuildComplexities(dependencies): + result = {} + def GetComplexity(p): + if p not in dependencies: + return 0 + if p not in result: + result[p] = 1 + sum(GetComplexity(x) for x in dependencies[p]) + return result[p] + for p in dependencies: + GetComplexity(p) + return result class TypeInferenceForRule: - def __init__(self, rule): + def __init__(self, rule, types_of_builtins): self.rule = rule self.variable_type = {} self.type_id_counter = 0 - self.types_of_builtins = types_of_builtins.TypesOfBultins() self.found_error = None + self.types_of_builtins = types_of_builtins def PerformInference(self): self.InitTypes() @@ -184,31 +247,39 @@ def MindPodLiterals(self): Walk(self.rule, ActMindingPodLiterals) def ActMindingBuiltinFieldTypes(self, node): + def InstillTypes(field_value, signature, output_value_type): + copier = reference_algebra.TypeStructureCopier() + copy = copier.CopyConcreteOrReferenceType + if output_value_type: + reference_algebra.Unify( + output_value_type, + copy(signature['logica_value'])) + for fv in field_value: + if fv['field'] in signature: + reference_algebra.Unify( + fv['value']['expression']['type']['the_type'], + copy(signature[fv['field']])) + for e in ExpressionsIterator(node): if 'call' in e: p = e['call']['predicate_name'] if p in self.types_of_builtins: - copier = reference_algebra.TypeStructureCopier() - copy = copier.CopyConcreteOrReferenceType + InstillTypes(e['call']['record']['field_value'], + self.types_of_builtins[p], + e['type']['the_type']) - reference_algebra.Unify( - e['type']['the_type'], - copy(self.types_of_builtins[p]['logica_value'])) - for fv in e['call']['record']['field_value']: - if fv['field'] in self.types_of_builtins[p]: - reference_algebra.Unify( - fv['value']['expression']['type']['the_type'], - copy(self.types_of_builtins[p][fv['field']])) if 'predicate' in node: p = node['predicate']['predicate_name'] if p in self.types_of_builtins: - copier = reference_algebra.TypeStructureCopier() - copy = copier.CopyConcreteOrReferenceType - for fv in node['predicate']['record']['field_value']: - if fv['field'] in self.types_of_builtins[p]: - reference_algebra.Unify( - fv['value']['expression']['type']['the_type'], - copy(self.types_of_builtins[p][fv['field']])) + InstillTypes(node['predicate']['record']['field_value'], + self.types_of_builtins[p], None) + + if 'head' in node: + p = node['head']['predicate_name'] + if p in self.types_of_builtins: + InstillTypes(node['head']['record']['field_value'], + self.types_of_builtins[p], None) + def MindBuiltinFieldTypes(self): Walk(self.rule, self.ActMindingBuiltinFieldTypes) @@ -252,17 +323,55 @@ def ActMindingInclusion(self, node): list_type, element_type ) + def ActMindingCombine(self, node): + if 'combine' in node: + field_value = node['combine']['head']['record']['field_value'] + [logica_value] = [fv['value'] + for fv in field_value + if fv['field'] == 'logica_value'] + reference_algebra.Unify( + node['type']['the_type'], + logica_value['aggregation']['expression']['type']['the_type'] + ) + + def IterateInference(self): Walk(self.rule, self.ActMindingRecordLiterals) Walk(self.rule, self.ActUnifying) Walk(self.rule, self.ActUnderstandingSubscription) Walk(self.rule, self.ActMindingListLiterals) Walk(self.rule, self.ActMindingInclusion) - + Walk(self.rule, self.ActMindingCombine) + +def RenderPredicateSignature(predicate_name, signature): + def FieldValue(f, v): + if isinstance(f, int): + field = '' + else: + field = f + ': ' + value = reference_algebra.RenderType(reference_algebra.VeryConcreteType(v)) + return field + value + field_values = [FieldValue(f, v) for f, v in signature.items() + if f != 'logica_value'] + signature_str = ', '.join(field_values) + maybe_value = [ + ' = ' + reference_algebra.RenderType( + reference_algebra.VeryConcreteType(v)) + for f, v in signature.items() if f == 'logica_value'] + value_or_nothing = maybe_value[0] if maybe_value else '' + result = f'type {predicate_name}({signature_str}){value_or_nothing};' + return result + + +class TypeErrorChecker: + def __init__(self, typed_rules): + self.typed_rules = typed_rules + + def CheckForError(self): self.found_error = self.SearchTypeErrors() if self.found_error.type_error: print(self.found_error.NiceMessage()) - + def SearchTypeErrors(self): found_error = ContextualizedError() def LookForError(node): @@ -277,5 +386,8 @@ def LookForError(node): found_error.ReplaceIfMoreUseful( t, node['expression_heritage'].Display(), v, node) - Walk(self.rule, LookForError) - return found_error + for rule in self.typed_rules: + Walk(rule, LookForError) + if found_error.type_error: + return found_error + return found_error \ No newline at end of file diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 0ad3ef4c..9097b7de 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -95,6 +95,8 @@ def RenderType(t): if isinstance(t, dict): return '{%s}' % ', '.join('%s: %s' % (k, RenderType(v)) for k, v in t.items()) + if isinstance(t, tuple): + return '(%s != %s)' % (RenderType(t[0]), RenderType(t[1])) def ConcreteType(t): if isinstance(t, TypeReference): diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 52a63471..ecec170c 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -75,6 +75,10 @@ def TypesOfBultins(): 'ArgMin': { 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), 'logica_value': x + }, + 'Range': { + 0: 'Num', + 'logica_value': ['Num'] } } return { From 8f1b6c0f823e4f83baf8c3d9954b39746eddff7a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 12 Jul 2023 18:54:37 -0700 Subject: [PATCH 046/141] Type inference research: combines. --- type_inference/research/infer.py | 44 ++- .../research/integration_tests/run_tests.py | 1 + .../typing_aggregation_test.txt | 6 +- .../integration_tests/typing_basic_test.txt | 48 +-- .../integration_tests/typing_combines_test.l | 19 ++ .../typing_combines_test.txt | 318 ++++++++++++++++++ .../integration_tests/typing_lists_test.txt | 30 +- .../integration_tests/typing_nested_test.txt | 26 +- 8 files changed, 426 insertions(+), 66 deletions(-) create mode 100644 type_inference/research/integration_tests/typing_combines_test.l create mode 100644 type_inference/research/integration_tests/typing_combines_test.txt diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 2121226c..bd13b34f 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -229,18 +229,13 @@ def GetTypeId(self): def ActInitializingTypes(self, node): for e in ExpressionsIterator(node): - i = self.GetTypeId() - if 'variable' in e: - var_name = e['variable']['var_name'] - use_type = self.variable_type.get( - var_name, - {'the_type': reference_algebra.TypeReference('Any'), 'type_id': i}) - self.variable_type[var_name] = use_type - else: - use_type = {'the_type': reference_algebra.TypeReference('Any'), 'type_id': i} - e['type'] = use_type + if 'variable' not in e: # Variables are convered separately. + e['type'] = { + 'the_type': reference_algebra.TypeReference('Any'), + 'type_id': self.GetTypeId()} def InitTypes(self): + WalkInitializingVariables(self.rule, self.GetTypeId) Walk(self.rule, self.ActInitializingTypes) def MindPodLiterals(self): @@ -390,4 +385,31 @@ def LookForError(node): Walk(rule, LookForError) if found_error.type_error: return found_error - return found_error \ No newline at end of file + return found_error + + +def WalkInitializingVariables(node, get_type): + """Initialize variables minding combines contexts.""" + type_of_variable = {} + def Jog(node): + nonlocal type_of_variable + if isinstance(node, list): + for v in node: + Jog(v) + if isinstance(node, dict): + if 'variable' in node: + var_name = node['variable']['var_name'] + if var_name not in type_of_variable: + type_of_variable[var_name] = { + 'the_type': reference_algebra.TypeReference('Any'), + 'type_id': get_type()} + node['type'] = type_of_variable[var_name] + for k in node: + if k != 'type': + if k != 'combine': + Jog(node[k]) + else: + backed_up_types = {k: v for k, v in type_of_variable.items()} + Jog(node[k]) + type_of_variable = backed_up_types + Jog(node) \ No newline at end of file diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index 3ead62fe..4e0069c5 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -30,6 +30,7 @@ def RunAll(): RunTypesTest('typing_aggregation_test') RunTypesTest('typing_lists_test') RunTypesTest('typing_nested_test') + RunTypesTest('typing_combines_test') if __name__ == '__main__': RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_aggregation_test.txt b/type_inference/research/integration_tests/typing_aggregation_test.txt index 564ec8b9..4cbb51b0 100644 --- a/type_inference/research/integration_tests/typing_aggregation_test.txt +++ b/type_inference/research/integration_tests/typing_aggregation_test.txt @@ -30,7 +30,7 @@ "expression_heritage": "t", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 1 }, "variable": { "var_name": "t" @@ -106,7 +106,7 @@ "expression_heritage": "t", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 1 }, "variable": { "var_name": "t" @@ -119,7 +119,7 @@ }, "type": { "the_type": "Num", - "type_id": 1 + "type_id": 2 } } } diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index 3e2d6737..5655e601 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -9,7 +9,7 @@ "expression_heritage": "x", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 0 }, "variable": { "var_name": "x" @@ -32,7 +32,7 @@ }, "type": { "the_type": "Num", - "type_id": 13 + "type_id": 14 } } } @@ -44,7 +44,7 @@ "expression_heritage": "z", "type": { "the_type": "Num", - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "z" @@ -58,7 +58,7 @@ "expression_heritage": "1 + z", "type": { "the_type": "Num", - "type_id": 12 + "type_id": 13 } } } @@ -75,7 +75,7 @@ "expression_heritage": "z", "type": { "the_type": "Num", - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "z" @@ -94,7 +94,7 @@ "b": "Num" } }, - "type_id": 5 + "type_id": 2 }, "variable": { "var_name": "w" @@ -111,7 +111,7 @@ "the_type": [ "Any" ], - "type_id": 17 + "type_id": 3 }, "variable": { "var_name": "l" @@ -129,7 +129,7 @@ "expression_heritage": "e", "type": { "the_type": "Any", - "type_id": 18 + "type_id": 4 }, "variable": { "var_name": "e" @@ -141,7 +141,7 @@ "the_type": [ "Any" ], - "type_id": 17 + "type_id": 3 }, "variable": { "var_name": "l" @@ -165,7 +165,7 @@ "b": "Num" } }, - "type_id": 5 + "type_id": 2 }, "variable": { "var_name": "w" @@ -183,7 +183,7 @@ "the_type": { "b": "Num" }, - "type_id": 22 + "type_id": 17 } }, "subscript": { @@ -196,7 +196,7 @@ }, "type": { "the_type": "Num", - "type_id": 20 + "type_id": 15 } }, "right_hand_side": { @@ -208,7 +208,7 @@ }, "type": { "the_type": "Num", - "type_id": 21 + "type_id": 16 } } } @@ -233,7 +233,7 @@ }, "type": { "the_type": "Num", - "type_id": 0 + "type_id": 5 } } } @@ -250,7 +250,7 @@ }, "type": { "the_type": "Str", - "type_id": 1 + "type_id": 6 } } } @@ -262,7 +262,7 @@ "expression_heritage": "x", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 0 }, "variable": { "var_name": "x" @@ -277,7 +277,7 @@ "expression_heritage": "z", "type": { "the_type": "Num", - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "z" @@ -299,7 +299,7 @@ "b": "Num" } }, - "type_id": 5 + "type_id": 2 }, "variable": { "var_name": "w" @@ -317,7 +317,7 @@ "the_type": { "b": "Num" }, - "type_id": 4 + "type_id": 7 } } } @@ -341,7 +341,7 @@ }, "type": { "the_type": "Num", - "type_id": 7 + "type_id": 9 } } } @@ -358,7 +358,7 @@ }, "type": { "the_type": "Str", - "type_id": 8 + "type_id": 10 } } } @@ -382,7 +382,7 @@ }, "type": { "the_type": "Num", - "type_id": 10 + "type_id": 12 } } } @@ -393,7 +393,7 @@ "the_type": { "z": "Num" }, - "type_id": 9 + "type_id": 11 } } } @@ -408,7 +408,7 @@ "t": "Num", "z": "Str" }, - "type_id": 6 + "type_id": 8 } } } diff --git a/type_inference/research/integration_tests/typing_combines_test.l b/type_inference/research/integration_tests/typing_combines_test.l new file mode 100644 index 00000000..67dbb6a2 --- /dev/null +++ b/type_inference/research/integration_tests/typing_combines_test.l @@ -0,0 +1,19 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test(x, y) :- + x += (a :- a in [1,2,3]), + y List= (a :- a in ["a", "b"]) diff --git a/type_inference/research/integration_tests/typing_combines_test.txt b/type_inference/research/integration_tests/typing_combines_test.txt new file mode 100644 index 00000000..cd2c6c0e --- /dev/null +++ b/type_inference/research/integration_tests/typing_combines_test.txt @@ -0,0 +1,318 @@ +[ + { + "body": { + "conjunction": { + "conjunct": [ + { + "unification": { + "left_hand_side": { + "expression_heritage": "x", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "x" + } + }, + "right_hand_side": { + "combine": { + "body": { + "conjunction": { + "conjunct": [ + { + "inclusion": { + "element": { + "expression_heritage": "a", + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "a" + } + }, + "list": { + "expression_heritage": "[1,2,3]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + }, + { + "expression_heritage": "2", + "literal": { + "the_number": { + "number": "2" + } + }, + "type": { + "the_type": "Num", + "type_id": 8 + } + }, + { + "expression_heritage": "3", + "literal": { + "the_number": { + "number": "3" + } + }, + "type": { + "the_type": "Num", + "type_id": 9 + } + } + ] + } + }, + "type": { + "the_type": [ + "Num" + ], + "type_id": 6 + } + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "x += (a :- a in [1,2,3])", + "head": { + "predicate_name": "Combine", + "record": { + "field_value": [ + { + "field": "logica_value", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "Agg+", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "a", + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "a" + } + } + } + } + ] + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + } + ] + } + } + }, + "type": { + "the_type": "Num", + "type_id": 4 + } + } + } + }, + { + "unification": { + "left_hand_side": { + "expression_heritage": "y", + "type": { + "the_type": [ + "Str" + ], + "type_id": 1 + }, + "variable": { + "var_name": "y" + } + }, + "right_hand_side": { + "combine": { + "body": { + "conjunction": { + "conjunct": [ + { + "inclusion": { + "element": { + "expression_heritage": "a", + "type": { + "the_type": "Str", + "type_id": 3 + }, + "variable": { + "var_name": "a" + } + }, + "list": { + "expression_heritage": "[\"a\", \"b\"]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "\"a\"", + "literal": { + "the_string": { + "the_string": "a" + } + }, + "type": { + "the_type": "Str", + "type_id": 13 + } + }, + { + "expression_heritage": "\"b\"", + "literal": { + "the_string": { + "the_string": "b" + } + }, + "type": { + "the_type": "Str", + "type_id": 14 + } + } + ] + } + }, + "type": { + "the_type": [ + "Str" + ], + "type_id": 12 + } + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "y List= (a :- a in [\"a\", \"b\"])", + "head": { + "predicate_name": "Combine", + "record": { + "field_value": [ + { + "field": "logica_value", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "List", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "a", + "type": { + "the_type": "Str", + "type_id": 3 + }, + "variable": { + "var_name": "a" + } + } + } + } + ] + } + }, + "type": { + "the_type": [ + "Str" + ], + "type_id": 11 + } + } + } + } + } + ] + } + } + }, + "type": { + "the_type": [ + "Str" + ], + "type_id": 10 + } + } + } + } + ] + } + }, + "full_text": "Test(x, y) :-\n x += (a :- a in [1,2,3]),\n y List= (a :- a in [\"a\", \"b\"])", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "x", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "x" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "y", + "type": { + "the_type": [ + "Str" + ], + "type_id": 1 + }, + "variable": { + "var_name": "y" + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_lists_test.txt b/type_inference/research/integration_tests/typing_lists_test.txt index 349716ac..504b09f3 100644 --- a/type_inference/research/integration_tests/typing_lists_test.txt +++ b/type_inference/research/integration_tests/typing_lists_test.txt @@ -9,7 +9,7 @@ "expression_heritage": "a", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 0 }, "variable": { "var_name": "a" @@ -29,7 +29,7 @@ }, "type": { "the_type": "Num", - "type_id": 7 + "type_id": 6 } }, { @@ -41,7 +41,7 @@ }, "type": { "the_type": "Num", - "type_id": 8 + "type_id": 7 } } ] @@ -51,7 +51,7 @@ "the_type": [ "Num" ], - "type_id": 6 + "type_id": 5 } } } @@ -70,7 +70,7 @@ "the_type": [ "Num" ], - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "b" @@ -88,7 +88,7 @@ "expression_heritage": "a", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 0 }, "variable": { "var_name": "a" @@ -100,7 +100,7 @@ "the_type": [ "Num" ], - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "b" @@ -116,7 +116,7 @@ "the_type": [ "Str" ], - "type_id": 4 + "type_id": 2 }, "variable": { "var_name": "c" @@ -136,7 +136,7 @@ }, "type": { "the_type": "Str", - "type_id": 14 + "type_id": 9 } } ] @@ -146,7 +146,7 @@ "the_type": [ "Str" ], - "type_id": 13 + "type_id": 8 } } } @@ -176,7 +176,7 @@ }, "type": { "the_type": "Num", - "type_id": 1 + "type_id": 4 } } ] @@ -186,7 +186,7 @@ "the_type": [ "Num" ], - "type_id": 0 + "type_id": 3 } } } @@ -198,7 +198,7 @@ "expression_heritage": "a", "type": { "the_type": "Num", - "type_id": 2 + "type_id": 0 }, "variable": { "var_name": "a" @@ -215,7 +215,7 @@ "the_type": [ "Num" ], - "type_id": 3 + "type_id": 1 }, "variable": { "var_name": "b" @@ -232,7 +232,7 @@ "the_type": [ "Str" ], - "type_id": 4 + "type_id": 2 }, "variable": { "var_name": "c" diff --git a/type_inference/research/integration_tests/typing_nested_test.txt b/type_inference/research/integration_tests/typing_nested_test.txt index adc97031..c9cb9946 100644 --- a/type_inference/research/integration_tests/typing_nested_test.txt +++ b/type_inference/research/integration_tests/typing_nested_test.txt @@ -76,7 +76,7 @@ }, "type": { "the_type": "Num", - "type_id": 7 + "type_id": 9 } } ] @@ -86,7 +86,7 @@ "the_type": [ "Num" ], - "type_id": 6 + "type_id": 8 } } } @@ -99,7 +99,7 @@ "Num" ] }, - "type_id": 5 + "type_id": 7 } } } @@ -114,7 +114,7 @@ ] } }, - "type_id": 4 + "type_id": 6 } } } @@ -131,7 +131,7 @@ } } }, - "type_id": 3 + "type_id": 5 } } ] @@ -149,7 +149,7 @@ } } ], - "type_id": 2 + "type_id": 4 } } } @@ -172,7 +172,7 @@ } } }, - "type_id": 8 + "type_id": 2 }, "variable": { "var_name": "x" @@ -214,7 +214,7 @@ "the_type": [ "Num" ], - "type_id": 10 + "type_id": 3 }, "variable": { "var_name": "c" @@ -246,7 +246,7 @@ } } }, - "type_id": 8 + "type_id": 2 }, "variable": { "var_name": "x" @@ -268,7 +268,7 @@ ] } }, - "type_id": 13 + "type_id": 12 } }, "subscript": { @@ -285,7 +285,7 @@ "Num" ] }, - "type_id": 12 + "type_id": 11 } }, "subscript": { @@ -300,7 +300,7 @@ "the_type": [ "Num" ], - "type_id": 11 + "type_id": 10 } } } @@ -327,7 +327,7 @@ "the_type": [ "Num" ], - "type_id": 10 + "type_id": 3 }, "variable": { "var_name": "c" From 08ab4c41f00df53b4f4216be83f9f8137a280a17 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 12 Jul 2023 19:37:26 -0700 Subject: [PATCH 047/141] Type inference: enable optional type checking --- logica.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/logica.py b/logica.py index 5631f71f..46a508cc 100755 --- a/logica.py +++ b/logica.py @@ -185,8 +185,6 @@ def main(argv): print(typing_engine.ShowPredicateTypes()) type_error_checker = infer.TypeErrorChecker(parsed_rules) type_error_checker.CheckForError() - # print(json.dumps(parsed_rules, sort_keys=True, indent=' ')) - return 0 predicates = argv[3] @@ -208,6 +206,10 @@ def main(argv): except functors.FunctorError as functor_exception: functor_exception.ShowMessage() sys.exit(1) + except infer.TypeErrorCaughtException as type_error_exception: + type_error_exception.ShowMessage() + sys.exit(1) + if command == 'print': print(formatted_sql) From f81f53373a603530645caedec1b34484bb0ef355 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 12 Jul 2023 19:37:33 -0700 Subject: [PATCH 048/141] Type inference: enable optional type checking --- compiler/universe.py | 16 ++++++++++++++-- type_inference/research/infer.py | 21 +++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/compiler/universe.py b/compiler/universe.py index fb4673ff..5ec4cc52 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -34,6 +34,7 @@ from compiler import functors from compiler import rule_translate from parser_py import parse + from type_inference.research import infer else: from ..common import color from ..compiler import dialects @@ -253,6 +254,14 @@ def Engine(self): AnnotationError('Unrecognized engine: %s' % engine, self.annotations['@Engine'][engine]) return engine + + def ShouldTypecheck(self): + if '@Engine' not in self.annotations: + return False + engine_annotation = list(self.annotations['@Engine'].values())[0] + if 'type_checking' not in engine_annotation: + return False + return engine_annotation['type_checking'] def ExtractSingleton(self, annotation_name, default_value): if not self.annotations[annotation_name]: @@ -495,7 +504,7 @@ def __init__(self, rules, table_aliases=None, user_flags=None): # Function compilation may have added irrelevant defines: self.execution = None - if False: + if self.annotations.ShouldTypecheck(): self.RunTypechecker() def UnfoldRecursion(self, rules): @@ -530,7 +539,10 @@ def RunTypechecker(self): Raises: TypeInferenceError if there are any type errors. """ - inference.CheckTypes(self.rules) + typing_engine = infer.TypesInferenceEngine(self.preparsed_rules) + typing_engine.InferTypes() + type_error_checker = infer.TypeErrorChecker(self.preparsed_rules) + type_error_checker.CheckForError(mode='raise') def RunMakes(self, rules): """Runs @Make instructions.""" diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index bd13b34f..6192b9a2 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + if '.' not in __package__: from type_inference.research import reference_algebra from type_inference.research import types_of_builtins @@ -358,14 +360,29 @@ def FieldValue(f, v): return result +class TypeErrorCaughtException(Exception): + """Exception thrown when user-error is detected at rule-compile time.""" + + def __init__(self, message): + super(TypeErrorCaughtException, self).__init__(message) + + def ShowMessage(self, stream=sys.stderr): + print(str(self), file=stream) + + class TypeErrorChecker: def __init__(self, typed_rules): self.typed_rules = typed_rules - def CheckForError(self): + def CheckForError(self, mode='print'): self.found_error = self.SearchTypeErrors() if self.found_error.type_error: - print(self.found_error.NiceMessage()) + if mode == 'print': + print(self.found_error.NiceMessage()) + elif mode == 'raise': + raise TypeErrorCaughtException(self.found_error.NiceMessage()) + else: + assert False def SearchTypeErrors(self): found_error = ContextualizedError() From 2bf96fe55a90000d73b1a0a2870c3733c4c2678a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:24:55 -0700 Subject: [PATCH 049/141] Type inference: enable psql records. --- type_inference/research/infer.py | 176 +++++++++++++++++++++++++++++-- 1 file changed, 167 insertions(+), 9 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 6192b9a2..f82788f2 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib import sys if '.' not in __package__: @@ -97,7 +98,9 @@ def ExpressionsIterator(node): for f in ExpressionFields(): if f in node: yield node[f] + if 'record' in node and 'field_value' not in node['record']: + # if 'record' in node and 'expression_heritage' in node: yield node['record'] if 'the_list' in node: for e in node['the_list']['element']: @@ -127,6 +130,9 @@ def ActMindingPodLiterals(node): if 'the_string' in e['literal']: reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Str')) +def ActClearingTypes(node): + if 'type' in node: + del node['type'] class TypesInferenceEngine: def __init__(self, parsed_rules): @@ -136,6 +142,12 @@ def __init__(self, parsed_rules): self.complexities = BuildComplexities(self.dependencies) self.parsed_rules = list(sorted(self.parsed_rules, key=lambda x: self.complexities[x['head']['predicate_name']])) self.types_of_builtins = types_of_builtins.TypesOfBultins() + self.typing_preamble = None + + def CollectTypes(self): + collector = TypeCollector(self.parsed_rules) + collector.CollectTypes() + self.typing_preamble = collector.typing_preamble def UpdateTypes(self, rule): predicate_name = rule['head']['predicate_name'] @@ -163,18 +175,16 @@ def UpdateTypes(self, rule): def InferTypes(self): for rule in self.parsed_rules: + if rule['head']['predicate_name'][0] == '@': + continue t = TypeInferenceForRule(rule, self.types_of_builtins) t.PerformInference() self.UpdateTypes(rule) - def Concretize(node): - if isinstance(node, dict): - if 'type' in node: - node['type']['the_type'] = reference_algebra.VeryConcreteType( - node['type']['the_type']) for rule in self.parsed_rules: - Walk(rule, Concretize) - + Walk(rule, ConcretizeTypes) + self.CollectTypes() + def ShowPredicateTypes(self): result_lines = [] for predicate_name, signature in self.types_of_builtins.items(): @@ -182,6 +192,13 @@ def ShowPredicateTypes(self): return '\n'.join(result_lines) +def ConcretizeTypes(node): + if isinstance(node, dict): + if 'type' in node: + node['type']['the_type'] = reference_algebra.VeryConcreteType( + node['type']['the_type']) + + def BuildDependencies(rules): def ExtractDendencies(rule): p = rule['head']['predicate_name'] @@ -203,6 +220,7 @@ def GetComplexity(p): if p not in dependencies: return 0 if p not in result: + result[p] = 1 result[p] = 1 + sum(GetComplexity(x) for x in dependencies[p]) return result[p] for p in dependencies: @@ -297,12 +315,18 @@ def ActUnderstandingSubscription(self, node): def ActMindingRecordLiterals(self, node): if 'type' in node and 'record' in node: + record_type = node['type']['the_type'] + reference_algebra.Unify( + record_type, + reference_algebra.TypeReference( + reference_algebra.OpenRecord())) for fv in node['record']['field_value']: - record_type = node['type']['the_type'] field_type = fv['value']['expression']['type']['the_type'] field_name = fv['field'] reference_algebra.UnifyRecordField( record_type, field_name, field_type) + # print('>>>', node) + node['type']['the_type'].CloseRecord() def ActMindingListLiterals(self, node): @@ -360,6 +384,76 @@ def FieldValue(f, v): return result +class TypeInferenceForStructure: + def __init__(self, structure, signatures): + self.structure = structure + self.signatures = signatures + + def PerformInference(self): + quazy_rule = self.BuildQuazyRule() + Walk(quazy_rule, ActClearingTypes) + inferencer = TypeInferenceForRule(quazy_rule, self.signatures) + inferencer.PerformInference() + + Walk(quazy_rule, ConcretizeTypes) + collector = TypeCollector([quazy_rule]) + collector.CollectTypes() + + import json + # + # print('>> quazy rule:', json.dumps(quazy_rule, indent=' ')) + + def BuildQuazyRule(self): + result = {} + result['quazy_body'] = self.BuildQuazyBody() + result['select'] = self.BuildSelect() + result['unnestings'] = self.BuildUnnestings() + return result + + def BuildUnnestings(self): + # print('>> unnestings', self.structure.unnestings) + result = [] + for variable, the_list in self.structure.unnestings: + result.append( + {'inclusion': { + 'element': variable, + 'list': the_list + }} + ) + return result + + def BuildQuazyBody(self): + # print('>> tables, vars_map: ', self.structure.tables, self.structure.vars_map) + calls = {} + + for table_id, predicate in self.structure.tables.items(): + calls[table_id] = { + 'predicate': {'predicate_name': predicate, + 'record': {'field_value': []}}} + + for (table_id, field), variable in self.structure.vars_map.items(): + if table_id is None: + # This is from unnestings. I don't recall why are those even here :-( + # Need to clarify. + continue + calls[table_id]['predicate']['record']['field_value'].append( + {'field': field, + 'value': {'expression': {'variable': {'var_name': variable}}} + } + ) + + return list(calls.values()) + + def BuildSelect(self): + field_values = [] + result = {'record': {'field_value': field_values}} + for k, v in self.structure.select.items(): + field_values.append({ + 'field': k, + 'value': {'expression': v} + }) + return result + class TypeErrorCaughtException(Exception): """Exception thrown when user-error is detected at rule-compile time.""" @@ -395,6 +489,7 @@ def LookForError(node): v = node['variable']['var_name'] else: v = None + assert 'expression_heritage' in node, node found_error.ReplaceIfMoreUseful( t, node['expression_heritage'].Display(), v, node) @@ -429,4 +524,67 @@ def Jog(node): backed_up_types = {k: v for k, v in type_of_variable.items()} Jog(node[k]) type_of_variable = backed_up_types - Jog(node) \ No newline at end of file + Jog(node) + +def Fingerprint(s): + return int(hashlib.md5(str(s).encode()).hexdigest()[:16], 16) + +def RecordTypeName(type_render): + return 'logicarecord%d' % (Fingerprint(type_render) % 1000000000) + +class TypeCollector: + def __init__(self, parsed_rules): + self.parsed_rules = parsed_rules + self.type_map = {} + self.psql_struct_type_name = {} + self.psql_type_definition = {} + self.definitions = [] + self.typing_preamble = '' + + def ActPopulatingTypeMap(self, node): + if 'type' in node: + t = node['type']['the_type'] + t_rendering = reference_algebra.RenderType(t) + self.type_map[t_rendering] = t + if isinstance(t, dict) and reference_algebra.IsFullyDefined(t): + node['type']['type_name'] = RecordTypeName(t_rendering) + + def CollectTypes(self): + Walk(self.parsed_rules, self.ActPopulatingTypeMap) + for t in self.type_map: + the_type = self.type_map[t] + if isinstance(the_type, dict): + if not reference_algebra.IsFullyDefined(the_type): + continue + self.psql_struct_type_name[t] = RecordTypeName(t) + self.BuildPsqlDefinitions() + + def PsqlType(self, t): + if t == 'Str': + return 'text' + if t == 'Num': + return 'numeric' + if isinstance(t, dict): + return self.psql_struct_type_name[reference_algebra.RenderType(t)] + if isinstance(t, list): + [e] = t + return self.PsqlType(e) + '[]' + assert False, t + + def BuildPsqlDefinitions(self): + for t in self.psql_struct_type_name: + args = ', '.join( + f + ' ' + self.PsqlType(v) + for f, v in sorted(self.type_map[t].items()) + ) + self.psql_type_definition[t] = f'create type %s as (%s);' % ( + self.psql_struct_type_name[t], args) + + wrap = lambda n, d: ( + f"if not exists (select 'I(am) :- I(think)' from pg_type where typname = '{n}') then {d} end if;" + ) + self.definitions = [ + wrap(self.psql_struct_type_name[t], self.psql_type_definition[t]) + for t in sorted(self.psql_struct_type_name, key=len)] + + self.typing_preamble = 'DO $$\nBEGIN\n' + '\n'.join(self.definitions) + '\nEND $$;\n' \ No newline at end of file From de389d791aae41f378b7db6d730e2282cc0ec728 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:25:00 -0700 Subject: [PATCH 050/141] Type inference: enable psql records. --- compiler/expr_translate.py | 18 +++++++- compiler/universe.py | 32 +++++++++++--- .../integration_tests/typing_basic_test.txt | 21 ++++++--- .../integration_tests/typing_nested_test.txt | 21 ++++++--- type_inference/research/reference_algebra.py | 21 ++++++++- type_inference/research/types_of_builtins.py | 44 +++++++++++++++++++ 6 files changed, 132 insertions(+), 25 deletions(-) diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 4b7ef5c9..fe391351 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -294,7 +294,7 @@ def RecordAsJson(self, record): value=self.ConvertToSql(f_v['value']['expression']))) return '{%s}' % ', '.join(json_field_values) - def Record(self, record): + def Record(self, record, record_type=None): if self.convert_to_json: return self.RecordAsJson(record) # TODO: Move this to dialects.py. @@ -310,6 +310,11 @@ def Record(self, record): for f_v in record['field_value']) if self.dialect.Name() == 'Trino': return '(SELECT %s)' % arguments_str + if self.dialect.Name() == 'PostgreSQL': + assert record_type, json.dumps(record, indent=' ') + args = ', '.join(self.ConvertToSql(f_v['value']['expression']) + for f_v in record['field_value']) + return 'ROW(%s)::%s' % (args, record_type) return 'STRUCT(%s)' % arguments_str def GenericSqlExpression(self, record): @@ -544,7 +549,16 @@ def ConvertToSql(self, expression): if 'record' in expression: record = expression['record'] - return self.Record(record) + record_type = expression.get('type', {}).get('type_name', None) + if self.dialect.Name() == 'PostgreSQL' and record_type is None: + raise self.exception_maker(color.Format( + 'Record needs type in PostgreSQL: ' + '{warning}{record}{end}.', dict( + record=expression['expression_heritage']))) + + return self.Record( + record, + record_type=record_type) if 'combine' in expression: return '(%s)' % ( diff --git a/compiler/universe.py b/compiler/universe.py index 5ec4cc52..80f462f7 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -193,7 +193,6 @@ def AttachedDatabases(self): result = {} for k, v in self.annotations['@AttachDatabase'].items(): if '1' not in v: - print('>>', v) AnnotationError('@AttachDatabase must have a single argument.', v) result[k] = v['1'] @@ -258,9 +257,16 @@ def Engine(self): def ShouldTypecheck(self): if '@Engine' not in self.annotations: return False + if len(self.annotations['@Engine'].values()) == 0: + return False + + engine = self.Engine() engine_annotation = list(self.annotations['@Engine'].values())[0] if 'type_checking' not in engine_annotation: - return False + if engine == 'psql': + return True + else: + return False return engine_annotation['type_checking'] def ExtractSingleton(self, annotation_name, default_value): @@ -499,14 +505,18 @@ def __init__(self, rules, table_aliases=None, user_flags=None): # We need to recompute annotations, because 'Make' created more rules and # annotations. self.annotations = Annotations(extended_rules, self.user_flags) + + # Infering types if requested. + self.typing_preamble = '' + self.predicate_signatures = {} + if self.annotations.ShouldTypecheck(): + self.typing_preamble = self.RunTypechecker() + # Build udfs, populating custom_udfs and custom_udf_definitions. self.BuildUdfs() # Function compilation may have added irrelevant defines: self.execution = None - if self.annotations.ShouldTypecheck(): - self.RunTypechecker() - def UnfoldRecursion(self, rules): annotations = Annotations(rules, {}) f = functors.Functors(rules) @@ -539,10 +549,13 @@ def RunTypechecker(self): Raises: TypeInferenceError if there are any type errors. """ - typing_engine = infer.TypesInferenceEngine(self.preparsed_rules) + rules = [r for _, r in self.rules] + typing_engine = infer.TypesInferenceEngine(rules) typing_engine.InferTypes() - type_error_checker = infer.TypeErrorChecker(self.preparsed_rules) + type_error_checker = infer.TypeErrorChecker(rules) type_error_checker.CheckForError(mode='raise') + self.predicate_signatures = typing_engine.types_of_builtins + return typing_engine.typing_preamble def RunMakes(self, rules): """Runs @Make instructions.""" @@ -763,6 +776,8 @@ def InitializeExecution(self, main_predicate): []) self.execution.dependencies_of = self.functors.args_of self.execution.dialect = dialects.Get(self.annotations.Engine()) + if self.execution.dialect.Name() == 'PostgreSQL': + self.execution.preamble += '\n' + self.typing_preamble def FormattedPredicateSql(self, name, allocator=None): """Printing top-level formatted SQL statement with defines and exports.""" @@ -932,6 +947,9 @@ def SingleRuleSql(self, rule, self.RunInjections(s, allocator) s.ElliminateInternalVariables(assert_full_ellimination=True) s.UnificationsToConstraints() + type_inference = infer.TypeInferenceForStructure(s, self.predicate_signatures) + type_inference.PerformInference() + try: sql = s.AsSql(self.MakeSubqueryTranslator(allocator), self.flag_values) except RuntimeError as runtime_error: diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index 5655e601..a0b4f8ed 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -94,7 +94,8 @@ "b": "Num" } }, - "type_id": 2 + "type_id": 2, + "type_name": "logicarecord715995786" }, "variable": { "var_name": "w" @@ -165,7 +166,8 @@ "b": "Num" } }, - "type_id": 2 + "type_id": 2, + "type_name": "logicarecord715995786" }, "variable": { "var_name": "w" @@ -183,7 +185,8 @@ "the_type": { "b": "Num" }, - "type_id": 17 + "type_id": 17, + "type_name": "logicarecord958681958" } }, "subscript": { @@ -299,7 +302,8 @@ "b": "Num" } }, - "type_id": 2 + "type_id": 2, + "type_name": "logicarecord715995786" }, "variable": { "var_name": "w" @@ -317,7 +321,8 @@ "the_type": { "b": "Num" }, - "type_id": 7 + "type_id": 7, + "type_name": "logicarecord958681958" } } } @@ -393,7 +398,8 @@ "the_type": { "z": "Num" }, - "type_id": 11 + "type_id": 11, + "type_name": "logicarecord574638620" } } } @@ -408,7 +414,8 @@ "t": "Num", "z": "Str" }, - "type_id": 8 + "type_id": 8, + "type_name": "logicarecord412758286" } } } diff --git a/type_inference/research/integration_tests/typing_nested_test.txt b/type_inference/research/integration_tests/typing_nested_test.txt index c9cb9946..63719904 100644 --- a/type_inference/research/integration_tests/typing_nested_test.txt +++ b/type_inference/research/integration_tests/typing_nested_test.txt @@ -99,7 +99,8 @@ "Num" ] }, - "type_id": 7 + "type_id": 7, + "type_name": "logicarecord137760342" } } } @@ -114,7 +115,8 @@ ] } }, - "type_id": 6 + "type_id": 6, + "type_name": "logicarecord261470720" } } } @@ -131,7 +133,8 @@ } } }, - "type_id": 5 + "type_id": 5, + "type_name": "logicarecord762067541" } } ] @@ -172,7 +175,8 @@ } } }, - "type_id": 2 + "type_id": 2, + "type_name": "logicarecord762067541" }, "variable": { "var_name": "x" @@ -246,7 +250,8 @@ } } }, - "type_id": 2 + "type_id": 2, + "type_name": "logicarecord762067541" }, "variable": { "var_name": "x" @@ -268,7 +273,8 @@ ] } }, - "type_id": 12 + "type_id": 12, + "type_name": "logicarecord261470720" } }, "subscript": { @@ -285,7 +291,8 @@ "Num" ] }, - "type_id": 11 + "type_id": 11, + "type_name": "logicarecord137760342" } }, "subscript": { diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 9097b7de..c3dd07f0 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -84,7 +84,9 @@ def CloseRecord(self): a = self while a.WeMustGoDeeper(): a = a.target - assert isinstance(a.target, dict) + if isinstance(a.target, BadType): + return + assert isinstance(a.target, dict), a.target a.target = ClosedRecord(a.target) def RenderType(t): @@ -94,7 +96,7 @@ def RenderType(t): return '[%s]' % RenderType(t[0]) if isinstance(t, dict): return '{%s}' % ', '.join('%s: %s' % (k, RenderType(v)) - for k, v in t.items()) + for k, v in sorted(t.items())) if isinstance(t, tuple): return '(%s != %s)' % (RenderType(t[0]), RenderType(t[1])) @@ -123,6 +125,21 @@ def VeryConcreteType(t): assert False +def IsFullyDefined(t): + if t == 'Any': + return False + if isinstance(t, str): + return True + if isinstance(t, BadType): + return False + if isinstance(t, list): + [e] = t + return IsFullyDefined(e) + if isinstance(t, dict): + return all(IsFullyDefined(v) for v in t.values()) + assert False + + def Rank(x): """Rank of the type, arbitrary order for sorting.""" x = ConcreteType(x) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index ecec170c..6367fba9 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -76,9 +76,53 @@ def TypesOfBultins(): 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), 'logica_value': x }, + 'ArgMax': { + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 'logica_value': x + }, + 'ArgMinK': { + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 1: 'Num', + 'logica_value': [x] + }, + 'ArgMaxK': { + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 1: 'Num', + 'logica_value': [x] + }, 'Range': { 0: 'Num', 'logica_value': ['Num'] + }, + 'Length': { + 0: 'Str', + 'logica_value': 'Num' + }, + 'Size': { + 0: ['Any'], + 'logica_value': 'Num' + }, + '-': { + 0: 'Num', + 'left': 'Num', + 'right': 'Num', + 'logica_value': 'Num' + }, + 'Min': { + 0: x, + 'logica_value': x + }, + 'Max': { + 0: x, + 'logica_value': y + }, + 'Array': { + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 'logica_value': y + }, + 'ValueOfUnnested': { + 0: x, + 'logica_value': x } } return { From a9342ff0a55184ffb49f28c20b7dad9366049e79 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:43:51 -0700 Subject: [PATCH 051/141] Type inference: enable psql records. --- compiler/universe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/universe.py b/compiler/universe.py index 80f462f7..9a596f33 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -42,6 +42,7 @@ from ..compiler import functors from ..compiler import rule_translate from ..parser_py import parse + from type_inference.research import infer PredicateInfo = collections.namedtuple('PredicateInfo', ['embeddable']) From b81bc9d6b892cac6926efe681043f7ee40dd7b84 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:44:28 -0700 Subject: [PATCH 052/141] Type inference: enable psql records. --- compiler/universe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/universe.py b/compiler/universe.py index 9a596f33..6d3963fa 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -42,7 +42,7 @@ from ..compiler import functors from ..compiler import rule_translate from ..parser_py import parse - from type_inference.research import infer + from ...type_inference.research import infer PredicateInfo = collections.namedtuple('PredicateInfo', ['embeddable']) From e421b768b7ea5e182443a3c067dfeb1a7e804db8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:44:44 -0700 Subject: [PATCH 053/141] Type inference: enable psql records. --- compiler/universe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/universe.py b/compiler/universe.py index 6d3963fa..bde8e644 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -42,7 +42,7 @@ from ..compiler import functors from ..compiler import rule_translate from ..parser_py import parse - from ...type_inference.research import infer + from ..type_inference.research import infer PredicateInfo = collections.namedtuple('PredicateInfo', ['embeddable']) From 8cf4582fdba1b13d487552cf2349485c4f29d438 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 15:27:33 -0700 Subject: [PATCH 054/141] Type inference: enable psql records. --- colab_logica.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index cfe4b34e..24f86a49 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -142,10 +142,12 @@ def RunSQL(sql, engine, connection=None, is_final=False): client = bigquery.Client(project=PROJECT) return client.query(sql).to_dataframe() elif engine == 'psql': + # Sorry, this is not looking good. + from sqlalchemy import text if is_final: - return pandas.read_sql(sql, connection) + return pandas.read_sql(text(sql), connection) else: - return connection.execute(sql) + return connection.execute(text(sql)) elif engine == 'sqlite': try: if is_final: From 7b0d0fb480792cbe0cb7fc41bf25ceb1e26115f8 Mon Sep 17 00:00:00 2001 From: EvgSkv Date: Fri, 14 Jul 2023 15:38:14 -0700 Subject: [PATCH 055/141] Adding structs test. --- integration_tests/psql_simple_structs_test.l | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 integration_tests/psql_simple_structs_test.l diff --git a/integration_tests/psql_simple_structs_test.l b/integration_tests/psql_simple_structs_test.l new file mode 100644 index 00000000..f4996c10 --- /dev/null +++ b/integration_tests/psql_simple_structs_test.l @@ -0,0 +1,24 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Test(x -> y, + {a: x, b: {c: [x, y]}, c: z}, + {a: x, b: {c: x, d: y}, some_field: x + y}) :- + x in [1, 2], + y in [3, 4], + z in [{a: "a"}, {b: "b"}]; + From 6dc666dec220afe787ebdbc1332ffeef87ffc1a2 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 16:30:04 -0700 Subject: [PATCH 056/141] Type inference: dealing with recursive types. --- integration_tests/psql_simple_structs_test.l | 2 +- type_inference/research/infer.py | 6 +++--- type_inference/research/reference_algebra.py | 19 +++++++++++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/integration_tests/psql_simple_structs_test.l b/integration_tests/psql_simple_structs_test.l index f4996c10..f007b3a6 100644 --- a/integration_tests/psql_simple_structs_test.l +++ b/integration_tests/psql_simple_structs_test.l @@ -20,5 +20,5 @@ Test(x -> y, {a: x, b: {c: x, d: y}, some_field: x + y}) :- x in [1, 2], y in [3, 4], - z in [{a: "a"}, {b: "b"}]; + z in [{a: "abc"}, {b: "def"}]; diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index f82788f2..a08ef2b5 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -85,9 +85,8 @@ def HelpfulErrorMessage(self): self.refers_to_variable) + result else: result = color.Format( - 'Expression {warning}%s{end} ' % - str(self.refers_to_expression['expression_heritage']) + result - ) + 'Expression {warning}{e}{end} ', + args_dict=dict(e=str(self.refers_to_expression['expression_heritage']))) + result return result @@ -193,6 +192,7 @@ def ShowPredicateTypes(self): def ConcretizeTypes(node): + # print('>> concretizing', node) if isinstance(node, dict): if 'type' in node: node['type']['the_type'] = reference_algebra.VeryConcreteType( diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index c3dd07f0..2e40ace1 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -99,6 +99,7 @@ def RenderType(t): for k, v in sorted(t.items())) if isinstance(t, tuple): return '(%s != %s)' % (RenderType(t[0]), RenderType(t[1])) + assert False, type(t) def ConcreteType(t): if isinstance(t, TypeReference): @@ -109,18 +110,24 @@ def ConcreteType(t): isinstance(t, str)) return t -def VeryConcreteType(t): +def VeryConcreteType(t, upward=None): + upward = upward or set() + if id(t) in upward: + return BadType(('...', '...')) + else: + upward = upward | set([id(t)]) + c = ConcreteType(t) if isinstance(c, BadType): - return BadType(VeryConcreteType(e) for e in c) + return BadType(VeryConcreteType(e, upward) for e in c) if isinstance(c, str): return c if isinstance(c, list): - return [VeryConcreteType(e) for e in c] + return [VeryConcreteType(e, upward) for e in c] if isinstance(c, dict): - return type(c)({f: VeryConcreteType(v) for f, v in c.items()}) + return type(c)({f: VeryConcreteType(v, upward) for f, v in c.items()}) assert False @@ -243,8 +250,8 @@ def Unify(a, b): if set(concrete_a) == set(concrete_b): UnifyFriendlyRecords(a, b, ClosedRecord) return - a.target = Incompatible(a, b) - b.target = Incompatible(b, a) + a.target = Incompatible(a.target, b.target) + b.target = Incompatible(b.target, a.target) return assert False assert False, (a, type(a)) From 04b891ca153c9000bf4384f12eb1dc6174f37747 Mon Sep 17 00:00:00 2001 From: EvgSkv Date: Fri, 14 Jul 2023 16:44:24 -0700 Subject: [PATCH 057/141] Type inference: struct ground test. --- .../psql_simple_structs_test.txt | 12 +++++++++ integration_tests/psql_structs_ground_test.l | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 integration_tests/psql_simple_structs_test.txt create mode 100644 integration_tests/psql_structs_ground_test.l diff --git a/integration_tests/psql_simple_structs_test.txt b/integration_tests/psql_simple_structs_test.txt new file mode 100644 index 00000000..6426f970 --- /dev/null +++ b/integration_tests/psql_simple_structs_test.txt @@ -0,0 +1,12 @@ + col0 | col1 | col2 +-------+---------------------------+--------------- + (1,3) | (1,"(""{1,3}"")","(abc)") | (1,"(1,3)",4) + (1,4) | (1,"(""{1,4}"")","(abc)") | (1,"(1,4)",5) + (2,3) | (2,"(""{2,3}"")","(abc)") | (2,"(2,3)",5) + (2,4) | (2,"(""{2,4}"")","(abc)") | (2,"(2,4)",6) + (1,3) | (1,"(""{1,3}"")","(def)") | (1,"(1,3)",4) + (1,4) | (1,"(""{1,4}"")","(def)") | (1,"(1,4)",5) + (2,3) | (2,"(""{2,3}"")","(def)") | (2,"(2,3)",5) + (2,4) | (2,"(""{2,4}"")","(def)") | (2,"(2,4)",6) +(8 rows) + diff --git a/integration_tests/psql_structs_ground_test.l b/integration_tests/psql_structs_ground_test.l new file mode 100644 index 00000000..2f1da497 --- /dev/null +++ b/integration_tests/psql_structs_ground_test.l @@ -0,0 +1,27 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +@Ground(Goal); +Goal("human" -> "happiness"); +Goal("bird" -> "flight"); +Goal("machine" -> "?"); +Goal("universe" -> "?"); + +@OrderBy(Test, "col0"); +Test(x) :- Goal(x -> "?"); + + From 1561a008d05b691103b510ee39055452abb6f93b Mon Sep 17 00:00:00 2001 From: EvgSkv Date: Fri, 14 Jul 2023 16:45:13 -0700 Subject: [PATCH 058/141] Type inference: struct ground test. --- integration_tests/psql_simple_structs_test.l | 2 +- integration_tests/run_tests.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/psql_simple_structs_test.l b/integration_tests/psql_simple_structs_test.l index f007b3a6..ab9db4dd 100644 --- a/integration_tests/psql_simple_structs_test.l +++ b/integration_tests/psql_simple_structs_test.l @@ -20,5 +20,5 @@ Test(x -> y, {a: x, b: {c: x, d: y}, some_field: x + y}) :- x in [1, 2], y in [3, 4], - z in [{a: "abc"}, {b: "def"}]; + z in [{a: "abc"}, {a: "def"}]; diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 66b7c2c5..e199da43 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -89,6 +89,8 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_structs_ground_test") + RunTest("psql_simple_structs_test") RunTest("psql_recursion_test") RunTest("psql_test") RunTest("psql_arg_min_test") From 64bb576083c4a24dff06ae96c8a26aa12bc66492 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 16:49:31 -0700 Subject: [PATCH 059/141] Type inference improvements. --- type_inference/research/infer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index a08ef2b5..850f01ad 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -408,6 +408,7 @@ def BuildQuazyRule(self): result['quazy_body'] = self.BuildQuazyBody() result['select'] = self.BuildSelect() result['unnestings'] = self.BuildUnnestings() + result['constraints'] = self.structure.constraints return result def BuildUnnestings(self): From 68b5a263f9c30069381e1729511fb02846fd0fe0 Mon Sep 17 00:00:00 2001 From: EvgSkv Date: Fri, 14 Jul 2023 16:56:36 -0700 Subject: [PATCH 060/141] Test inference: improving nuances. --- integration_tests/psql_structs_ground_test.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 integration_tests/psql_structs_ground_test.txt diff --git a/integration_tests/psql_structs_ground_test.txt b/integration_tests/psql_structs_ground_test.txt new file mode 100644 index 00000000..a25d5026 --- /dev/null +++ b/integration_tests/psql_structs_ground_test.txt @@ -0,0 +1,6 @@ + col0 +---------- + machine + universe +(2 rows) + From adb7de633dcb353b24fff5b430ce16868e8f3f09 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 16:59:39 -0700 Subject: [PATCH 061/141] Type inference improvements. --- integration_tests/psql_arg_min_max_k_test.l | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/integration_tests/psql_arg_min_max_k_test.l b/integration_tests/psql_arg_min_max_k_test.l index d0f35d77..13a54948 100755 --- a/integration_tests/psql_arg_min_max_k_test.l +++ b/integration_tests/psql_arg_min_max_k_test.l @@ -30,13 +30,8 @@ TestArgMin() ArgMin= payload -> v :- Data(v:, payload:); ArgMax3(x) = ArgMaxK(x, 3); ArgMin2(x) = ArgMinK(x, 2); -# Type is created as follows: -# create type StringIntPair as (first text, second decimal) -StringIntPair(s, i) = SqlExpr("({s}, {i})::StringIntPair", - {s:, i:}); - -TestArgMaxK() ArgMax3= StringIntPair(payload, v) -> v :- Data(v:, payload:); -TestArgMinK() ArgMin2= StringIntPair(payload, v) -> v :- Data(v:, payload:); +TestArgMaxK() ArgMax3= {payload:, v:} -> v :- Data(v:, payload:); +TestArgMinK() ArgMin2= {payload:, v:} -> v :- Data(v:, payload:); @OrderBy(Test, "arg_opt"); Test(opt: "Max", arg_opt: TestArgMax(), arg_opt_k: TestArgMaxK()); From 53c4061bf791f91106579e4f378f58bfcbe9fb68 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 17:10:13 -0700 Subject: [PATCH 062/141] Type inference improvements. --- compiler/expr_translate.py | 5 +++-- integration_tests/psql_pair_test.l | 12 ++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index fe391351..450f1b14 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -312,8 +312,9 @@ def Record(self, record, record_type=None): return '(SELECT %s)' % arguments_str if self.dialect.Name() == 'PostgreSQL': assert record_type, json.dumps(record, indent=' ') - args = ', '.join(self.ConvertToSql(f_v['value']['expression']) - for f_v in record['field_value']) + args = ', '.join( + self.ConvertToSql(f_v['value']['expression']) + for f_v in sorted(record['field_value'], key=lambda x: x['field'])) return 'ROW(%s)::%s' % (args, record_type) return 'STRUCT(%s)' % arguments_str diff --git a/integration_tests/psql_pair_test.l b/integration_tests/psql_pair_test.l index f8883350..20b75d99 100644 --- a/integration_tests/psql_pair_test.l +++ b/integration_tests/psql_pair_test.l @@ -7,17 +7,13 @@ Word("water"); Word("wind"); Word("sun"); -# We assume there exists StringNumPair -# (first text, second numeric) type. -StringNumPair(s, i) = SqlExpr("({s}, {i})::StringNumPair", {s:, i:}); - -WordAndLength(StringNumPair(word, Length(word))) :- +WordAndLength({word:, length: Length(word)}) :- Word(word); -WordsByLengthList() Array= length -> StringNumPair(word, length) :- +WordsByLengthList() Array= length -> {word:, length:} :- WordAndLength(word_and_length), - word == word_and_length.first, - length == word_and_length.second; + word == word_and_length.word, + length == word_and_length.length; Test := WordsByLengthList() # If test fails create the type by running: From bab8821b187e183d2d1e402c440aafe060d5edaa Mon Sep 17 00:00:00 2001 From: EvgSkv Date: Fri, 14 Jul 2023 17:13:30 -0700 Subject: [PATCH 063/141] Type inference improvements. --- integration_tests/psql_pair_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/psql_pair_test.txt b/integration_tests/psql_pair_test.txt index d279e313..c09f3aa2 100644 --- a/integration_tests/psql_pair_test.txt +++ b/integration_tests/psql_pair_test.txt @@ -1,5 +1,5 @@ logica_value ----------------------------------------------- - {"(sun,3)","(fire,4)","(wind,4)","(water,5)"} + {"(3,sun)","(4,fire)","(4,wind)","(5,water)"} (1 row) From df26ddd0dd62bd3bcada070deb70c3fd5af3931c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 17:23:56 -0700 Subject: [PATCH 064/141] Type inference improvements. --- type_inference/research/infer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 850f01ad..6c68f50b 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -91,7 +91,8 @@ def HelpfulErrorMessage(self): def ExpressionFields(): - return ['expression', 'left_hand_side', 'right_hand_side'] + return ['expression', 'left_hand_side', 'right_hand_side', + 'condition', 'consequence', 'otherwise'] def ExpressionsIterator(node): for f in ExpressionFields(): From 21cf4f4b0e52950bb04774ba5b87d582c0cbcf1c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 14 Jul 2023 17:38:18 -0700 Subject: [PATCH 065/141] Type inference improvements. --- type_inference/research/infer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 6c68f50b..ad9637e3 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -356,6 +356,17 @@ def ActMindingCombine(self, node): logica_value['aggregation']['expression']['type']['the_type'] ) + def ActMindingImplications(self, node): + if 'implication' in node: + for if_then in node['implication']['if_then']: + reference_algebra.Unify( + node['type']['the_type'], + if_then['consequence']['type']['the_type'] + ) + reference_algebra.Unify( + node['type']['the_type'], + node['implication']['otherwise']['type']['the_type'] + ) def IterateInference(self): Walk(self.rule, self.ActMindingRecordLiterals) @@ -364,6 +375,7 @@ def IterateInference(self): Walk(self.rule, self.ActMindingListLiterals) Walk(self.rule, self.ActMindingInclusion) Walk(self.rule, self.ActMindingCombine) + Walk(self.rule, self.ActMindingImplications) def RenderPredicateSignature(predicate_name, signature): def FieldValue(f, v): From 8b4f80cf0ac829856995de6bddd11b56d910091a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 17 Jul 2023 13:52:53 -0400 Subject: [PATCH 066/141] Type inference improvements: combines. --- integration_tests/sqlite_array_sub_test.l | 2 +- integration_tests/sqlite_funcs_test.l | 2 +- integration_tests/sqlite_unwrapping_test.l | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/integration_tests/sqlite_array_sub_test.l b/integration_tests/sqlite_array_sub_test.l index c917acf4..06425b7f 100644 --- a/integration_tests/sqlite_array_sub_test.l +++ b/integration_tests/sqlite_array_sub_test.l @@ -17,7 +17,7 @@ # Please never use it this way. # This feature is provided for Spanner indexes specification. -@Engine("sqlite"); +@Engine("sqlite", type_checking: false); Books() = [ { diff --git a/integration_tests/sqlite_funcs_test.l b/integration_tests/sqlite_funcs_test.l index 0c5f06d8..ad888634 100644 --- a/integration_tests/sqlite_funcs_test.l +++ b/integration_tests/sqlite_funcs_test.l @@ -15,7 +15,7 @@ # Testing misc functions of SQLite. -@Engine("sqlite"); +@Engine("sqlite", type_checking: false); Test("Set") = Size(s) :- s Set= (x + y :- x in Range(5), y in Range(5)); Test("Sort") = Sort([6,3,2,5,1,4]); diff --git a/integration_tests/sqlite_unwrapping_test.l b/integration_tests/sqlite_unwrapping_test.l index f6b05d74..3af658a7 100644 --- a/integration_tests/sqlite_unwrapping_test.l +++ b/integration_tests/sqlite_unwrapping_test.l @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -@Engine("sqlite"); +@Engine("sqlite", type_checking: false); + Test("simple_assignment", {a:, b:}) :- {a:, b:} == {a: "va", b: "vb"}; Test("arrow_assignment", {a:, b:}) :- a -> b == "va" -> "vb"; From 874751765f76efcde8bfa54b951e1f94ecc29f0a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 17 Jul 2023 13:52:57 -0400 Subject: [PATCH 067/141] Type inference improvements: combines. --- integration_tests/sqlite_functors_test.l | 2 +- integration_tests/sqlite_record_assembler.l | 2 +- type_inference/research/infer.py | 52 +- .../research/integration_tests/run_tests.py | 2 + .../integration_tests/typing_combine2_test.l | 21 + .../typing_combine2_test.txt | 326 ++++++ .../typing_kitchensync_test.l | 42 + .../typing_kitchensync_test.txt | 947 ++++++++++++++++++ type_inference/research/reference_algebra.py | 11 + 9 files changed, 1392 insertions(+), 13 deletions(-) create mode 100644 type_inference/research/integration_tests/typing_combine2_test.l create mode 100644 type_inference/research/integration_tests/typing_combine2_test.txt create mode 100644 type_inference/research/integration_tests/typing_kitchensync_test.l create mode 100644 type_inference/research/integration_tests/typing_kitchensync_test.txt diff --git a/integration_tests/sqlite_functors_test.l b/integration_tests/sqlite_functors_test.l index 38797638..dcd07c08 100644 --- a/integration_tests/sqlite_functors_test.l +++ b/integration_tests/sqlite_functors_test.l @@ -16,7 +16,7 @@ # Testing subtle functor application order. @Engine("sqlite"); -T1("t1"); +T1() = "t1"; F() = T1(); D := F(); B() = D(); diff --git a/integration_tests/sqlite_record_assembler.l b/integration_tests/sqlite_record_assembler.l index 0a1b2988..f7842eb4 100644 --- a/integration_tests/sqlite_record_assembler.l +++ b/integration_tests/sqlite_record_assembler.l @@ -15,7 +15,7 @@ # Testing SQLite record introspection. -@Engine("sqlite"); +@Engine("sqlite", type_checking: false); Test(1, AssembleRecord(["planet" -> "Earth", "species" -> "Humans"])); Test(2, DisassembleRecord({planet: "Mars", species: "Apes"})); diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index ad9637e3..2c789afd 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -15,6 +15,7 @@ # limitations under the License. import hashlib +import json import sys if '.' not in __package__: @@ -46,6 +47,8 @@ def Replace(self, type_error, context_string, refers_to_variable, refers_to_expr def ReplaceIfMoreUseful(self, type_error, context_string, refers_to_variable, refers_to_expression): if self.type_error is None: self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) + if self.context_string == 'UNKNOWN LOCATION': + self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) elif self.refers_to_variable is None and refers_to_variable: self.Replace(type_error, context_string, refers_to_variable, refers_to_expression) elif 'literal' in self.refers_to_expression: @@ -134,6 +137,19 @@ def ActClearingTypes(node): if 'type' in node: del node['type'] +def ActRememberingTypes(node): + if 'type' in node: + node['remembered_type'] = json.dumps(node['type']['the_type']) + +def ActRecallingTypes(node): + if 'remembered_type' in node: + remembered_type = reference_algebra.Revive( + json.loads(node['remembered_type'])) + reference_algebra.Unify( + node['type']['the_type'], + remembered_type + ) + class TypesInferenceEngine: def __init__(self, parsed_rules): self.parsed_rules = parsed_rules @@ -404,9 +420,11 @@ def __init__(self, structure, signatures): def PerformInference(self): quazy_rule = self.BuildQuazyRule() + Walk(quazy_rule, ActRememberingTypes) Walk(quazy_rule, ActClearingTypes) inferencer = TypeInferenceForRule(quazy_rule, self.signatures) inferencer.PerformInference() + Walk(quazy_rule, ActRecallingTypes) Walk(quazy_rule, ConcretizeTypes) collector = TypeCollector([quazy_rule]) @@ -503,10 +521,16 @@ def LookForError(node): v = node['variable']['var_name'] else: v = None - assert 'expression_heritage' in node, node - found_error.ReplaceIfMoreUseful( - t, node['expression_heritage'].Display(), v, - node) + # Combines don't have expression_heritage. + # assert 'expression_heritage' in node, (node, t) + if 'expression_heritage' not in node: + found_error.ReplaceIfMoreUseful( + t, 'UNKNOWN LOCATION', v, + node) + else: + found_error.ReplaceIfMoreUseful( + t, node['expression_heritage'].Display(), v, + node) for rule in self.typed_rules: Walk(rule, LookForError) if found_error.type_error: @@ -517,11 +541,11 @@ def LookForError(node): def WalkInitializingVariables(node, get_type): """Initialize variables minding combines contexts.""" type_of_variable = {} - def Jog(node): + def Jog(node, found_combines): nonlocal type_of_variable if isinstance(node, list): for v in node: - Jog(v) + Jog(v, found_combines) if isinstance(node, dict): if 'variable' in node: var_name = node['variable']['var_name'] @@ -533,12 +557,18 @@ def Jog(node): for k in node: if k != 'type': if k != 'combine': - Jog(node[k]) + Jog(node[k], found_combines) else: - backed_up_types = {k: v for k, v in type_of_variable.items()} - Jog(node[k]) - type_of_variable = backed_up_types - Jog(node) + found_combines.append(node[k]) + def JogPredicate(node): + nonlocal type_of_variable + found_combines = [] + Jog(node, found_combines) + backed_up_types = {k: v for k, v in type_of_variable.items()} + for n in found_combines: + JogPredicate(n) + type_of_variable = backed_up_types + JogPredicate(node) def Fingerprint(s): return int(hashlib.md5(str(s).encode()).hexdigest()[:16], 16) diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index 4e0069c5..fd209807 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -26,11 +26,13 @@ def RunTypesTest(name, src=None, golden=None): logica_test.TestManager.RunTypesTest(name, src, golden) def RunAll(): + RunTypesTest('typing_kitchensync_test') RunTypesTest('typing_basic_test') RunTypesTest('typing_aggregation_test') RunTypesTest('typing_lists_test') RunTypesTest('typing_nested_test') RunTypesTest('typing_combines_test') + RunTypesTest('typing_combine2_test') if __name__ == '__main__': RunAll() \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_combine2_test.l b/type_inference/research/integration_tests/typing_combine2_test.l new file mode 100644 index 00000000..ba80c451 --- /dev/null +++ b/type_inference/research/integration_tests/typing_combine2_test.l @@ -0,0 +1,21 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +T(1); + +Test(x,{o:1}) :- x List= ({a:} :- a in [{z:}]), T(z); \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_combine2_test.txt b/type_inference/research/integration_tests/typing_combine2_test.txt new file mode 100644 index 00000000..ed4a4434 --- /dev/null +++ b/type_inference/research/integration_tests/typing_combine2_test.txt @@ -0,0 +1,326 @@ +[ + { + "full_text": "@Engine(\"psql\")", + "head": { + "predicate_name": "@Engine", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "\"psql\"", + "literal": { + "the_string": { + "the_string": "psql" + } + } + } + } + } + ] + } + } + }, + { + "full_text": "T(1)", + "head": { + "predicate_name": "T", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 0 + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "unification": { + "left_hand_side": { + "expression_heritage": "x", + "type": { + "the_type": [ + { + "a": { + "z": "Num" + } + } + ], + "type_id": 0 + }, + "variable": { + "var_name": "x" + } + }, + "right_hand_side": { + "combine": { + "body": { + "conjunction": { + "conjunct": [ + { + "inclusion": { + "element": { + "expression_heritage": "a", + "type": { + "the_type": { + "z": "Num" + }, + "type_id": 2, + "type_name": "logicarecord574638620" + }, + "variable": { + "var_name": "a" + } + }, + "list": { + "expression_heritage": "[{z:}]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "{z:}", + "record": { + "field_value": [ + { + "field": "z", + "value": { + "expression": { + "expression_heritage": "z", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "z" + } + } + } + } + ] + }, + "type": { + "the_type": { + "z": "Num" + }, + "type_id": 9, + "type_name": "logicarecord574638620" + } + } + ] + } + }, + "type": { + "the_type": [ + { + "z": "Num" + } + ], + "type_id": 8 + } + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "x List= ({a:} :- a in [{z:}])", + "head": { + "predicate_name": "Combine", + "record": { + "field_value": [ + { + "field": "logica_value", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "List", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "{a:}", + "record": { + "field_value": [ + { + "field": "a", + "value": { + "expression": { + "expression_heritage": "a", + "type": { + "the_type": { + "z": "Num" + }, + "type_id": 2, + "type_name": "logicarecord574638620" + }, + "variable": { + "var_name": "a" + } + } + } + } + ] + }, + "type": { + "the_type": { + "a": { + "z": "Num" + } + }, + "type_id": 7, + "type_name": "logicarecord350574256" + } + } + } + } + ] + } + }, + "type": { + "the_type": [ + { + "a": { + "z": "Num" + } + } + ], + "type_id": 6 + } + } + } + } + } + ] + } + } + }, + "type": { + "the_type": [ + { + "a": { + "z": "Num" + } + } + ], + "type_id": 5 + } + } + } + }, + { + "predicate": { + "predicate_name": "T", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "z", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "z" + } + } + } + } + ] + } + } + } + ] + } + }, + "full_text": "Test(x,{o:1}) :- x List= ({a:} :- a in [{z:}]), T(z)", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "x", + "type": { + "the_type": [ + { + "a": { + "z": "Num" + } + } + ], + "type_id": 0 + }, + "variable": { + "var_name": "x" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "{o:1}", + "record": { + "field_value": [ + { + "field": "o", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 4 + } + } + } + } + ] + }, + "type": { + "the_type": { + "o": "Num" + }, + "type_id": 3, + "type_name": "logicarecord944799139" + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_kitchensync_test.l b/type_inference/research/integration_tests/typing_kitchensync_test.l new file mode 100644 index 00000000..5109f44a --- /dev/null +++ b/type_inference/research/integration_tests/typing_kitchensync_test.l @@ -0,0 +1,42 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Kitchensync( + name: "sync 1", + items: [{item: "plate", num: 10}, {item: "cup", num: 6}]); +Kitchensync( + name: "sync 2", + items: [{item: "plate", num: 15}, {item: "spoon", num: 20}]); +Kitchensync( + name: "luxury sync", + items: [{item: "fork", num: 5}, {item: "cup", num: 4}]); + +Test(name:, overview:) :- + Kitchensync(name:, items:), + Kitchensync(name:, items:), + overview List= ( + {item:, quantity:} :- + quantity = ( + if num > 9 then + "large" + else + "small" + ), + num > 5, + {item:, num:} in items + ); \ No newline at end of file diff --git a/type_inference/research/integration_tests/typing_kitchensync_test.txt b/type_inference/research/integration_tests/typing_kitchensync_test.txt new file mode 100644 index 00000000..a3e922be --- /dev/null +++ b/type_inference/research/integration_tests/typing_kitchensync_test.txt @@ -0,0 +1,947 @@ +[ + { + "full_text": "@Engine(\"psql\")", + "head": { + "predicate_name": "@Engine", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "\"psql\"", + "literal": { + "the_string": { + "the_string": "psql" + } + } + } + } + } + ] + } + } + }, + { + "full_text": "Kitchensync(\n name: \"sync 1\", \n items: [{item: \"plate\", num: 10}, {item: \"cup\", num: 6}])", + "head": { + "predicate_name": "Kitchensync", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "\"sync 1\"", + "literal": { + "the_string": { + "the_string": "sync 1" + } + }, + "type": { + "the_type": "Str", + "type_id": 0 + } + } + } + }, + { + "field": "items", + "value": { + "expression": { + "expression_heritage": "[{item: \"plate\", num: 10}, {item: \"cup\", num: 6}]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "{item: \"plate\", num: 10}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"plate\"", + "literal": { + "the_string": { + "the_string": "plate" + } + }, + "type": { + "the_type": "Str", + "type_id": 4 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "10", + "literal": { + "the_number": { + "number": "10" + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 2, + "type_name": "logicarecord454966611" + } + }, + { + "expression_heritage": "{item: \"cup\", num: 6}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"cup\"", + "literal": { + "the_string": { + "the_string": "cup" + } + }, + "type": { + "the_type": "Str", + "type_id": 6 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "6", + "literal": { + "the_number": { + "number": "6" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 3, + "type_name": "logicarecord454966611" + } + } + ] + } + }, + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 1 + } + } + } + } + ] + } + } + }, + { + "full_text": "Kitchensync(\n name: \"sync 2\", \n items: [{item: \"plate\", num: 15}, {item: \"spoon\", num: 20}])", + "head": { + "predicate_name": "Kitchensync", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "\"sync 2\"", + "literal": { + "the_string": { + "the_string": "sync 2" + } + }, + "type": { + "the_type": "Str", + "type_id": 0 + } + } + } + }, + { + "field": "items", + "value": { + "expression": { + "expression_heritage": "[{item: \"plate\", num: 15}, {item: \"spoon\", num: 20}]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "{item: \"plate\", num: 15}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"plate\"", + "literal": { + "the_string": { + "the_string": "plate" + } + }, + "type": { + "the_type": "Str", + "type_id": 4 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "15", + "literal": { + "the_number": { + "number": "15" + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 2, + "type_name": "logicarecord454966611" + } + }, + { + "expression_heritage": "{item: \"spoon\", num: 20}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"spoon\"", + "literal": { + "the_string": { + "the_string": "spoon" + } + }, + "type": { + "the_type": "Str", + "type_id": 6 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "20", + "literal": { + "the_number": { + "number": "20" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 3, + "type_name": "logicarecord454966611" + } + } + ] + } + }, + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 1 + } + } + } + } + ] + } + } + }, + { + "full_text": "Kitchensync(\n name: \"luxury sync\", \n items: [{item: \"fork\", num: 5}, {item: \"cup\", num: 4}])", + "head": { + "predicate_name": "Kitchensync", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "\"luxury sync\"", + "literal": { + "the_string": { + "the_string": "luxury sync" + } + }, + "type": { + "the_type": "Str", + "type_id": 0 + } + } + } + }, + { + "field": "items", + "value": { + "expression": { + "expression_heritage": "[{item: \"fork\", num: 5}, {item: \"cup\", num: 4}]", + "literal": { + "the_list": { + "element": [ + { + "expression_heritage": "{item: \"fork\", num: 5}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"fork\"", + "literal": { + "the_string": { + "the_string": "fork" + } + }, + "type": { + "the_type": "Str", + "type_id": 4 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "5", + "literal": { + "the_number": { + "number": "5" + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 2, + "type_name": "logicarecord454966611" + } + }, + { + "expression_heritage": "{item: \"cup\", num: 4}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "\"cup\"", + "literal": { + "the_string": { + "the_string": "cup" + } + }, + "type": { + "the_type": "Str", + "type_id": 6 + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "4", + "literal": { + "the_number": { + "number": "4" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 3, + "type_name": "logicarecord454966611" + } + } + ] + } + }, + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 1 + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "Kitchensync", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "name", + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "name" + } + } + } + }, + { + "field": "items", + "value": { + "expression": { + "expression_heritage": "items", + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 2 + }, + "variable": { + "var_name": "items" + } + } + } + } + ] + } + } + }, + { + "predicate": { + "predicate_name": "Kitchensync", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "name", + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "name" + } + } + } + }, + { + "field": "items", + "value": { + "expression": { + "expression_heritage": "items", + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 2 + }, + "variable": { + "var_name": "items" + } + } + } + } + ] + } + } + }, + { + "unification": { + "left_hand_side": { + "expression_heritage": "overview", + "type": { + "the_type": [ + { + "item": "Str", + "quantity": "Str" + } + ], + "type_id": 1 + }, + "variable": { + "var_name": "overview" + } + }, + "right_hand_side": { + "combine": { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "=", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "quantity", + "type": { + "the_type": "Str", + "type_id": 4 + }, + "variable": { + "var_name": "quantity" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "if num > 9 then\n \"large\"\n else\n \"small\"", + "implication": { + "if_then": [ + { + "condition": { + "call": { + "predicate_name": ">", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "num", + "type": { + "the_type": "Num", + "type_id": 5 + }, + "variable": { + "var_name": "num" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "9", + "literal": { + "the_number": { + "number": "9" + } + }, + "type": { + "the_type": "Num", + "type_id": 13 + } + } + } + } + ] + } + }, + "expression_heritage": "num > 9", + "type": { + "the_type": "Any", + "type_id": 11 + } + }, + "consequence": { + "expression_heritage": "\"large\"", + "literal": { + "the_string": { + "the_string": "large" + } + }, + "type": { + "the_type": "Str", + "type_id": 12 + } + } + } + ], + "otherwise": { + "expression_heritage": "\"small\"", + "literal": { + "the_string": { + "the_string": "small" + } + }, + "type": { + "the_type": "Str", + "type_id": 10 + } + } + }, + "type": { + "the_type": "Str", + "type_id": 9 + } + } + } + } + ] + } + } + }, + { + "predicate": { + "predicate_name": ">", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "num", + "type": { + "the_type": "Num", + "type_id": 5 + }, + "variable": { + "var_name": "num" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "5", + "literal": { + "the_number": { + "number": "5" + } + }, + "type": { + "the_type": "Num", + "type_id": 14 + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "expression_heritage": "{item:, num:}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "item", + "type": { + "the_type": "Str", + "type_id": 3 + }, + "variable": { + "var_name": "item" + } + } + } + }, + { + "field": "num", + "value": { + "expression": { + "expression_heritage": "num", + "type": { + "the_type": "Num", + "type_id": 5 + }, + "variable": { + "var_name": "num" + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "num": "Num" + }, + "type_id": 15, + "type_name": "logicarecord454966611" + } + }, + "list": { + "expression_heritage": "items", + "type": { + "the_type": [ + { + "item": "Str", + "num": "Num" + } + ], + "type_id": 2 + }, + "variable": { + "var_name": "items" + } + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "overview List= (\n {item:, quantity:} :-\n quantity = (\n if num > 9 then\n \"large\"\n else\n \"small\"\n ),\n num > 5,\n {item:, num:} in items\n )", + "head": { + "predicate_name": "Combine", + "record": { + "field_value": [ + { + "field": "logica_value", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "List", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "{item:, quantity:}", + "record": { + "field_value": [ + { + "field": "item", + "value": { + "expression": { + "expression_heritage": "item", + "type": { + "the_type": "Str", + "type_id": 3 + }, + "variable": { + "var_name": "item" + } + } + } + }, + { + "field": "quantity", + "value": { + "expression": { + "expression_heritage": "quantity", + "type": { + "the_type": "Str", + "type_id": 4 + }, + "variable": { + "var_name": "quantity" + } + } + } + } + ] + }, + "type": { + "the_type": { + "item": "Str", + "quantity": "Str" + }, + "type_id": 8, + "type_name": "logicarecord980116590" + } + } + } + } + ] + } + }, + "type": { + "the_type": [ + { + "item": "Str", + "quantity": "Str" + } + ], + "type_id": 7 + } + } + } + } + } + ] + } + } + }, + "type": { + "the_type": [ + { + "item": "Str", + "quantity": "Str" + } + ], + "type_id": 6 + } + } + } + } + ] + } + }, + "full_text": "Test(name:, overview:) :-\n Kitchensync(name:, items:),\n Kitchensync(name:, items:),\n overview List= (\n {item:, quantity:} :-\n quantity = (\n if num > 9 then\n \"large\"\n else\n \"small\"\n ),\n num > 5,\n {item:, num:} in items\n )", + "head": { + "predicate_name": "Test", + "record": { + "field_value": [ + { + "field": "name", + "value": { + "expression": { + "expression_heritage": "name", + "type": { + "the_type": "Str", + "type_id": 0 + }, + "variable": { + "var_name": "name" + } + } + } + }, + { + "field": "overview", + "value": { + "expression": { + "expression_heritage": "overview", + "type": { + "the_type": [ + { + "item": "Str", + "quantity": "Str" + } + ], + "type_id": 1 + }, + "variable": { + "var_name": "overview" + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 2e40ace1..80d9d52e 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -314,3 +314,14 @@ def CopyTypeReference(self, t): n = TypeReference(target) self.id_to_reference[id(t)] = n return self.id_to_reference[id(t)] + +def Revive(t): + if isinstance(t, str): + return TypeReference(t) + if isinstance(t, dict): + return TypeReference(OpenRecord({k: Revive(v) for k, v in t.items()})) + if isinstance(t, list): + return TypeReference(list(map(Revive, t))) + if isinstance(t, BadType): + return TypeReference(BadType(map(Revive, t))) + assert False, [type(t), t] \ No newline at end of file From 51c084d89e72a89c1a904ad802bc92f05f1f15f3 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Mon, 17 Jul 2023 18:46:26 +0000 Subject: [PATCH 068/141] Type inference: testing. --- integration_tests/psql_record_combine_test.l | 21 +++++++++++++++++++ .../psql_record_combine_test.txt | 5 +++++ integration_tests/run_tests.py | 1 + 3 files changed, 27 insertions(+) create mode 100644 integration_tests/psql_record_combine_test.l create mode 100644 integration_tests/psql_record_combine_test.txt diff --git a/integration_tests/psql_record_combine_test.l b/integration_tests/psql_record_combine_test.l new file mode 100644 index 00000000..f49c7525 --- /dev/null +++ b/integration_tests/psql_record_combine_test.l @@ -0,0 +1,21 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +T(1); + +Test(x, {o:1}) :- x List= ({a:} :- a in [{z:}]), T(z); + diff --git a/integration_tests/psql_record_combine_test.txt b/integration_tests/psql_record_combine_test.txt new file mode 100644 index 00000000..6014ba8b --- /dev/null +++ b/integration_tests/psql_record_combine_test.txt @@ -0,0 +1,5 @@ + col0 | col1 +---------------+------ + {"(\"(1)\")"} | (1) +(1 row) + diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index e199da43..0c20e8e7 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -89,6 +89,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_record_combine_test") RunTest("psql_structs_ground_test") RunTest("psql_simple_structs_test") RunTest("psql_recursion_test") From ef7dcf802989b1b8db626f87ea3bbaa83c92c73d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 17 Jul 2023 15:42:49 -0400 Subject: [PATCH 069/141] Type inference: annotate arrays in psql. --- compiler/expr_translate.py | 18 ++++++++-- integration_tests/psql_combine_test.l | 33 +++++++++++++++++++ integration_tests/run_tests.py | 1 + type_inference/research/infer.py | 5 ++- .../typing_combine2_test.txt | 5 +++ .../typing_combines_test.txt | 6 ++++ .../typing_kitchensync_test.txt | 10 ++++++ .../integration_tests/typing_lists_test.txt | 8 +++++ .../integration_tests/typing_nested_test.txt | 7 ++++ 9 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 integration_tests/psql_combine_test.l diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 450f1b14..9c457155 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -254,8 +254,13 @@ def ListLiteralInternals(self, literal): return ', '.join([self.ConvertToSql(e) for e in literal['element']]) - def ListLiteral(self, literal): - return self.dialect.ArrayPhrase() % self.ListLiteralInternals(literal) + def ListLiteral(self, literal, element_type_name): + suffix = ('::' + element_type_name + '[]' + if self.dialect.Name() == 'PostgreSQL' + else '') + return ( + self.dialect.ArrayPhrase() % + self.ListLiteralInternals(literal)) + suffix def BoolLiteral(self, literal): return literal['the_bool'] @@ -439,7 +444,14 @@ def ConvertToSql(self, expression): if 'the_string' in literal: return self.StrLiteral(literal['the_string']) if 'the_list' in literal: - return self.ListLiteral(literal['the_list']) + the_list = literal['the_list'] + element_type = expression.get('type', {}).get('element_type_name', None) + if self.dialect.Name() == 'PostgreSQL' and element_type is None: + raise self.exception_maker(color.Format( + 'Array needs type in PostgreSQL: ' + '{warning}{the_list}{end}.', dict( + the_list=expression['expression_heritage']))) + return self.ListLiteral(the_list, element_type) if 'the_bool' in literal: return self.BoolLiteral(literal['the_bool']) if 'the_null' in literal: diff --git a/integration_tests/psql_combine_test.l b/integration_tests/psql_combine_test.l new file mode 100644 index 00000000..6b623641 --- /dev/null +++ b/integration_tests/psql_combine_test.l @@ -0,0 +1,33 @@ +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Testing combine expressions that could be ambigous in SQLite. +@Engine("psql"); + +T(1); +T(2); +T(3); +T(4); + +@With(R); +R(x, l) :- T(x), l == (if x == 2 || x == 3 then [x] else []); + +P1(x, y) :- T(x), y += x; +P2(x, col1? List= y) distinct :- T(x), y in [1,2,3,4], R(x, l), ~(y in l); + +# Test("1", x, y) :- P1(x, y); +# Test("2", x, y) :- P2(x, y); + +Test() = (if true then [1] else []); \ No newline at end of file diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 0c20e8e7..436fc5f3 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -89,6 +89,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_combine_test") RunTest("psql_record_combine_test") RunTest("psql_structs_ground_test") RunTest("psql_simple_structs_test") diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 2c789afd..a2d8132a 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -592,6 +592,9 @@ def ActPopulatingTypeMap(self, node): self.type_map[t_rendering] = t if isinstance(t, dict) and reference_algebra.IsFullyDefined(t): node['type']['type_name'] = RecordTypeName(t_rendering) + if isinstance(t, list) and reference_algebra.IsFullyDefined(t): + [e] = t + node['type']['element_type_name'] = self.PsqlType(e) def CollectTypes(self): Walk(self.parsed_rules, self.ActPopulatingTypeMap) @@ -609,7 +612,7 @@ def PsqlType(self, t): if t == 'Num': return 'numeric' if isinstance(t, dict): - return self.psql_struct_type_name[reference_algebra.RenderType(t)] + return RecordTypeName(reference_algebra.RenderType(t)) if isinstance(t, list): [e] = t return self.PsqlType(e) + '[]' diff --git a/type_inference/research/integration_tests/typing_combine2_test.txt b/type_inference/research/integration_tests/typing_combine2_test.txt index ed4a4434..9523c9ad 100644 --- a/type_inference/research/integration_tests/typing_combine2_test.txt +++ b/type_inference/research/integration_tests/typing_combine2_test.txt @@ -58,6 +58,7 @@ "left_hand_side": { "expression_heritage": "x", "type": { + "element_type_name": "logicarecord350574256", "the_type": [ { "a": { @@ -129,6 +130,7 @@ } }, "type": { + "element_type_name": "logicarecord574638620", "the_type": [ { "z": "Num" @@ -200,6 +202,7 @@ } }, "type": { + "element_type_name": "logicarecord350574256", "the_type": [ { "a": { @@ -218,6 +221,7 @@ } }, "type": { + "element_type_name": "logicarecord350574256", "the_type": [ { "a": { @@ -268,6 +272,7 @@ "expression": { "expression_heritage": "x", "type": { + "element_type_name": "logicarecord350574256", "the_type": [ { "a": { diff --git a/type_inference/research/integration_tests/typing_combines_test.txt b/type_inference/research/integration_tests/typing_combines_test.txt index cd2c6c0e..cd25ca89 100644 --- a/type_inference/research/integration_tests/typing_combines_test.txt +++ b/type_inference/research/integration_tests/typing_combines_test.txt @@ -77,6 +77,7 @@ } }, "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -145,6 +146,7 @@ "left_hand_side": { "expression_heritage": "y", "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -204,6 +206,7 @@ } }, "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -249,6 +252,7 @@ } }, "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -263,6 +267,7 @@ } }, "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -300,6 +305,7 @@ "expression": { "expression_heritage": "y", "type": { + "element_type_name": "text", "the_type": [ "Str" ], diff --git a/type_inference/research/integration_tests/typing_kitchensync_test.txt b/type_inference/research/integration_tests/typing_kitchensync_test.txt index a3e922be..79d44d1d 100644 --- a/type_inference/research/integration_tests/typing_kitchensync_test.txt +++ b/type_inference/research/integration_tests/typing_kitchensync_test.txt @@ -155,6 +155,7 @@ } }, "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -303,6 +304,7 @@ } }, "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -451,6 +453,7 @@ } }, "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -496,6 +499,7 @@ "expression": { "expression_heritage": "items", "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -540,6 +544,7 @@ "expression": { "expression_heritage": "items", "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -563,6 +568,7 @@ "left_hand_side": { "expression_heritage": "overview", "type": { + "element_type_name": "logicarecord980116590", "the_type": [ { "item": "Str", @@ -783,6 +789,7 @@ "list": { "expression_heritage": "items", "type": { + "element_type_name": "logicarecord454966611", "the_type": [ { "item": "Str", @@ -869,6 +876,7 @@ } }, "type": { + "element_type_name": "logicarecord980116590", "the_type": [ { "item": "Str", @@ -886,6 +894,7 @@ } }, "type": { + "element_type_name": "logicarecord980116590", "the_type": [ { "item": "Str", @@ -926,6 +935,7 @@ "expression": { "expression_heritage": "overview", "type": { + "element_type_name": "logicarecord980116590", "the_type": [ { "item": "Str", diff --git a/type_inference/research/integration_tests/typing_lists_test.txt b/type_inference/research/integration_tests/typing_lists_test.txt index 504b09f3..3a0cfdd5 100644 --- a/type_inference/research/integration_tests/typing_lists_test.txt +++ b/type_inference/research/integration_tests/typing_lists_test.txt @@ -48,6 +48,7 @@ } }, "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -67,6 +68,7 @@ "expression": { "expression_heritage": "b", "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -97,6 +99,7 @@ "list": { "expression_heritage": "b", "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -113,6 +116,7 @@ "left_hand_side": { "expression_heritage": "c", "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -143,6 +147,7 @@ } }, "type": { + "element_type_name": "text", "the_type": [ "Str" ], @@ -183,6 +188,7 @@ } }, "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -212,6 +218,7 @@ "expression": { "expression_heritage": "b", "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -229,6 +236,7 @@ "expression": { "expression_heritage": "c", "type": { + "element_type_name": "text", "the_type": [ "Str" ], diff --git a/type_inference/research/integration_tests/typing_nested_test.txt b/type_inference/research/integration_tests/typing_nested_test.txt index 63719904..631d58e7 100644 --- a/type_inference/research/integration_tests/typing_nested_test.txt +++ b/type_inference/research/integration_tests/typing_nested_test.txt @@ -14,6 +14,7 @@ "expression": { "expression_heritage": "l", "type": { + "element_type_name": "logicarecord762067541", "the_type": [ { "a": { @@ -83,6 +84,7 @@ } }, "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -141,6 +143,7 @@ } }, "type": { + "element_type_name": "logicarecord762067541", "the_type": [ { "a": { @@ -185,6 +188,7 @@ "list": { "expression_heritage": "l", "type": { + "element_type_name": "logicarecord762067541", "the_type": [ { "a": { @@ -215,6 +219,7 @@ "expression": { "expression_heritage": "c", "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -304,6 +309,7 @@ } }, "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], @@ -331,6 +337,7 @@ "list": { "expression_heritage": "c", "type": { + "element_type_name": "numeric", "the_type": [ "Num" ], From 08037db5e0aefe1f9cd86388d2b6676777bb16fc Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 17 Jul 2023 16:03:33 -0400 Subject: [PATCH 070/141] Type inference: annotate arrays in psql. --- compiler/dialects.py | 2 +- compiler/rule_translate.py | 2 +- integration_tests/psql_combine_test.l | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index aa23d8bd..932afbb3 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -224,7 +224,7 @@ def ArrayPhrase(self): return 'ARRAY[%s]' def GroupBySpecBy(self): - return 'name' + return 'expr' def DecorateCombineRule(self, rule, var): return rule diff --git a/compiler/rule_translate.py b/compiler/rule_translate.py index d2072917..a0c36025 100755 --- a/compiler/rule_translate.py +++ b/compiler/rule_translate.py @@ -508,7 +508,7 @@ def AsSql(self, subquery_encoder=None, flag_values=None): subquery_encoder.execution.dialect.UnnestPhrase().format( ql.ConvertToSql(the_list), ql.ConvertToSql(element))) if not tables: - tables.append('(SELECT "singleton" as s) as unused_singleton') + tables.append("(SELECT 'singleton' as s) as unused_singleton") from_str = ', '.join(tables) # Indent the from_str. from_str = '\n'.join(' ' + l for l in from_str.split('\n')) diff --git a/integration_tests/psql_combine_test.l b/integration_tests/psql_combine_test.l index 6b623641..e5433521 100644 --- a/integration_tests/psql_combine_test.l +++ b/integration_tests/psql_combine_test.l @@ -27,7 +27,5 @@ R(x, l) :- T(x), l == (if x == 2 || x == 3 then [x] else []); P1(x, y) :- T(x), y += x; P2(x, col1? List= y) distinct :- T(x), y in [1,2,3,4], R(x, l), ~(y in l); -# Test("1", x, y) :- P1(x, y); -# Test("2", x, y) :- P2(x, y); - -Test() = (if true then [1] else []); \ No newline at end of file +Test("1", x, y) :- P1(x, y); +Test("2", x, y) :- P2(x, y); \ No newline at end of file From 8ed2fa35eb80a7475c9d6f4092da6373dec53882 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Mon, 17 Jul 2023 20:20:03 +0000 Subject: [PATCH 071/141] Type inference: testing. --- integration_tests/psql_combine_test.l | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/psql_combine_test.l b/integration_tests/psql_combine_test.l index e5433521..ed0b2616 100644 --- a/integration_tests/psql_combine_test.l +++ b/integration_tests/psql_combine_test.l @@ -24,8 +24,8 @@ T(4); @With(R); R(x, l) :- T(x), l == (if x == 2 || x == 3 then [x] else []); -P1(x, y) :- T(x), y += x; +P1(x, y) :- T(x), y List= (i :- i in Range(x)); P2(x, col1? List= y) distinct :- T(x), y in [1,2,3,4], R(x, l), ~(y in l); -Test("1", x, y) :- P1(x, y); -Test("2", x, y) :- P2(x, y); \ No newline at end of file +Test("test1", x, y) :- P1(x, y); +Test("test2", x, y) :- P2(x, y); \ No newline at end of file From bc79550c8e8c0dcc2de4b2a35f28038de0e2b54d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Mon, 17 Jul 2023 20:20:18 +0000 Subject: [PATCH 072/141] Type inference: testing. --- integration_tests/psql_combine_test.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 integration_tests/psql_combine_test.txt diff --git a/integration_tests/psql_combine_test.txt b/integration_tests/psql_combine_test.txt new file mode 100644 index 00000000..76f81a3e --- /dev/null +++ b/integration_tests/psql_combine_test.txt @@ -0,0 +1,12 @@ + col0 | col1 | col2 +-------+------+----------- + test1 | 1 | {0} + test1 | 2 | {0,1} + test1 | 3 | {0,1,2} + test1 | 4 | {0,1,2,3} + test2 | 1 | {1,2,3,4} + test2 | 2 | {1,3,4} + test2 | 3 | {1,2,4} + test2 | 4 | {1,2,3,4} +(8 rows) + From 7e28250fff4e9cf5270e36af73a39a08bc87f83b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Mon, 17 Jul 2023 21:27:21 +0000 Subject: [PATCH 073/141] Improving PSQL combine. --- compiler/dialects.py | 142 ++++++++++++------------ integration_tests/psql_combine_test.l | 2 +- integration_tests/psql_combine_test.txt | 8 +- 3 files changed, 77 insertions(+), 75 deletions(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index 932afbb3..3e5928d2 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -100,74 +100,7 @@ def BuiltInFunctions(self): } def DecorateCombineRule(self, rule, var): - """Resolving ambiguity of aggregation scope.""" - # Entangling result of aggregation with a variable that comes from a list - # unnested inside a combine expression, to make it clear that aggregation - # must be done in the combine. - rule = copy.deepcopy(rule) - - rule['head']['record']['field_value'][0]['value'][ - 'aggregation']['expression']['call'][ - 'record']['field_value'][0]['value'] = ( - { - 'expression': { - 'call': { - 'predicate_name': 'MagicalEntangle', - 'record': { - 'field_value': [ - { - 'field': 0, - 'value': rule['head']['record']['field_value'][0]['value'][ - 'aggregation']['expression']['call'][ - 'record']['field_value'][0]['value'] - }, - { - 'field': 1, - 'value': { - 'expression': { - 'variable': { - 'var_name': var - } - } - } - } - ] - } - } - } - } - ) - - if 'body' not in rule: - rule['body'] = {'conjunction': {'conjunct': []}} - rule['body']['conjunction']['conjunct'].append( - { - "inclusion": { - "list": { - "literal": { - "the_list": { - "element": [ - { - "literal": { - "the_number": { - "number": "0" - } - } - } - ] - } - } - }, - "element": { - "variable": { - "var_name": var - } - } - } - } - ) - return rule - + return DecorateCombineRule(rule, var) def InfixOperators(self): return { @@ -203,7 +136,8 @@ def BuiltInFunctions(self): 'ToString': 'CAST(%s AS TEXT)', 'Element': '({0})[{1} + 1]', 'Size': 'ARRAY_LENGTH(%s, 1)', - 'Count': 'COUNT(DISTINCT {0})' + 'Count': 'COUNT(DISTINCT {0})', + 'MagicalEntangle': '(CASE WHEN {1} = 0 THEN {0} ELSE NULL END)' } def InfixOperators(self): @@ -227,7 +161,7 @@ def GroupBySpecBy(self): return 'expr' def DecorateCombineRule(self, rule, var): - return rule + return DecorateCombineRule(rule, var) class Trino(Dialect): @@ -307,6 +241,74 @@ def GroupBySpecBy(self): def DecorateCombineRule(self, rule, var): return rule +def DecorateCombineRule(rule, var): + """Resolving ambiguity of aggregation scope.""" + # Entangling result of aggregation with a variable that comes from a list + # unnested inside a combine expression, to make it clear that aggregation + # must be done in the combine. + rule = copy.deepcopy(rule) + + rule['head']['record']['field_value'][0]['value'][ + 'aggregation']['expression']['call'][ + 'record']['field_value'][0]['value'] = ( + { + 'expression': { + 'call': { + 'predicate_name': 'MagicalEntangle', + 'record': { + 'field_value': [ + { + 'field': 0, + 'value': rule['head']['record']['field_value'][0]['value'][ + 'aggregation']['expression']['call'][ + 'record']['field_value'][0]['value'] + }, + { + 'field': 1, + 'value': { + 'expression': { + 'variable': { + 'var_name': var + } + } + } + } + ] + } + } + } + } + ) + + if 'body' not in rule: + rule['body'] = {'conjunction': {'conjunct': []}} + rule['body']['conjunction']['conjunct'].append( + { + "inclusion": { + "list": { + "literal": { + "the_list": { + "element": [ + { + "literal": { + "the_number": { + "number": "0" + } + } + } + ] + } + } + }, + "element": { + "variable": { + "var_name": var + } + } + } + } + ) + return rule DIALECTS = { 'bigquery': BigQueryDialect, diff --git a/integration_tests/psql_combine_test.l b/integration_tests/psql_combine_test.l index ed0b2616..cb3e2413 100644 --- a/integration_tests/psql_combine_test.l +++ b/integration_tests/psql_combine_test.l @@ -24,7 +24,7 @@ T(4); @With(R); R(x, l) :- T(x), l == (if x == 2 || x == 3 then [x] else []); -P1(x, y) :- T(x), y List= (i :- i in Range(x)); +P1(x, y) :- T(x), y List= x; P2(x, col1? List= y) distinct :- T(x), y in [1,2,3,4], R(x, l), ~(y in l); Test("test1", x, y) :- P1(x, y); diff --git a/integration_tests/psql_combine_test.txt b/integration_tests/psql_combine_test.txt index 76f81a3e..7f1c990e 100644 --- a/integration_tests/psql_combine_test.txt +++ b/integration_tests/psql_combine_test.txt @@ -1,9 +1,9 @@ col0 | col1 | col2 -------+------+----------- - test1 | 1 | {0} - test1 | 2 | {0,1} - test1 | 3 | {0,1,2} - test1 | 4 | {0,1,2,3} + test1 | 1 | {1} + test1 | 2 | {2} + test1 | 3 | {3} + test1 | 4 | {4} test2 | 1 | {1,2,3,4} test2 | 2 | {1,3,4} test2 | 3 | {1,2,4} From e3973d9be94d94aafac843fb14bafec0d2136c18 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Mon, 17 Jul 2023 23:04:05 +0000 Subject: [PATCH 074/141] Type inference testing. --- integration_tests/psql_purchase2_test.l | 76 ++++++++++++++++++++ integration_tests/psql_purchase2_test.txt | 8 +++ integration_tests/psql_purchase3_test.l | 69 ++++++++++++++++++ integration_tests/psql_purchase_test.l | 69 ++++++++++++++++++ integration_tests/psql_purchase_test.txt | 11 +++ integration_tests/run_tests.py | 2 + type_inference/research/types_of_builtins.py | 4 ++ 7 files changed, 239 insertions(+) create mode 100644 integration_tests/psql_purchase2_test.l create mode 100644 integration_tests/psql_purchase2_test.txt create mode 100644 integration_tests/psql_purchase3_test.l create mode 100644 integration_tests/psql_purchase_test.l create mode 100644 integration_tests/psql_purchase_test.txt diff --git a/integration_tests/psql_purchase2_test.l b/integration_tests/psql_purchase2_test.l new file mode 100644 index 00000000..c259a2c9 --- /dev/null +++ b/integration_tests/psql_purchase2_test.l @@ -0,0 +1,76 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Items(item: "Soap", price: 20); +Items(item: "Milk", price: 10); +Items(item: "Bread", price: 5); +Items(item: "Coffee", price: 7); +Items(item: "Firewood", price: 15); + +MoreExpensiveThan(item1) List= item2 :- + Items(item: item1, price: price1), + Items(item: item2, price: price2), + price1 > price2; + +BuyEvent(purchase_id: 1, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 2, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 3, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 3, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 4, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 4, item: "Soap", quantity: 1); +BuyEvent(purchase_id: 5, item: "Milk", quantity: 4); +BuyEvent(purchase_id: 5, item: "Bread", quantity: 1); +BuyEvent(purchase_id: 5, item: "Coffee", quantity: 2); +BuyEvent(purchase_id: 6, item: "Firewood", quantity: 1); +BuyEvent(purchase_id: 6, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 7, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 7, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 7, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 8, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 8, item: "Soap", quantity: 1); + +Buyer(buyer_id: 11, purchase_id: 1); +Buyer(buyer_id: 12, purchase_id: 2); +Buyer(buyer_id: 13, purchase_id: 3); +Buyer(buyer_id: 14, purchase_id: 4); +Buyer(buyer_id: 12, purchase_id: 5); +Buyer(buyer_id: 13, purchase_id: 6); +Buyer(buyer_id: 14, purchase_id: 7); +Buyer(buyer_id: 11, purchase_id: 8); + +@OrderBy(Purchase, "purchase_id"); +Purchase(purchase_id:, items:, expensive_items:, buyer_id:) :- + Buyer(buyer_id:, purchase_id:), + items List= ( + {item:, quantity:, price:} :- + BuyEvent(purchase_id:, item:, quantity:), + Items(item:, price:) + ), + expensive_items List= ( + {item:, more_expensive_than: MoreExpensiveThan(item)} :- + item_record in items, + item = item_record.item + ); + +@OrderBy(Test, "buyer_id"); +Test(buyer_id:, purchases:) :- + buyers Set= (b :- Buyer(buyer_id: b)), + buyer_id in buyers, + purchases List= ( + {purchase: items} :- + Purchase(items:, buyer_id:) + ); diff --git a/integration_tests/psql_purchase2_test.txt b/integration_tests/psql_purchase2_test.txt new file mode 100644 index 00000000..0397f5b8 --- /dev/null +++ b/integration_tests/psql_purchase2_test.txt @@ -0,0 +1,8 @@ + buyer_id | purchases +----------+-------------------------------------------------------------------------------------------------------------------------------- + 11 | {"(\"{\"\"(Soap,20,3)\"\"}\")","(\"{\"\"(Soap,20,1)\"\",\"\"(Firewood,15,5)\"\"}\")"} + 12 | {"(\"{\"\"(Milk,10,1)\"\"}\")","(\"{\"\"(Milk,10,4)\"\",\"\"(Bread,5,1)\"\",\"\"(Coffee,7,2)\"\"}\")"} + 13 | {"(\"{\"\"(Bread,5,2)\"\",\"\"(Coffee,7,1)\"\"}\")","(\"{\"\"(Soap,20,3)\"\",\"\"(Firewood,15,1)\"\"}\")"} + 14 | {"(\"{\"\"(Soap,20,1)\"\",\"\"(Firewood,15,5)\"\"}\")","(\"{\"\"(Milk,10,1)\"\",\"\"(Bread,5,2)\"\",\"\"(Coffee,7,1)\"\"}\")"} +(4 rows) + diff --git a/integration_tests/psql_purchase3_test.l b/integration_tests/psql_purchase3_test.l new file mode 100644 index 00000000..213a5387 --- /dev/null +++ b/integration_tests/psql_purchase3_test.l @@ -0,0 +1,69 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Items(item: "Soap", price: 20); +Items(item: "Milk", price: 10); +Items(item: "Bread", price: 5); +Items(item: "Coffee", price: 7); +Items(item: "Firewood", price: 15); + +MoreExpensiveThan(item1) List= item2 :- + Items(item: item1, price: price1), + Items(item: item2, price: price2), + price1 > price2; + +BuyEvent(purchase_id: 1, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 2, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 3, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 3, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 4, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 4, item: "Soap", quantity: 1); +BuyEvent(purchase_id: 5, item: "Milk", quantity: 4); +BuyEvent(purchase_id: 5, item: "Bread", quantity: 1); +BuyEvent(purchase_id: 5, item: "Coffee", quantity: 2); +BuyEvent(purchase_id: 6, item: "Firewood", quantity: 1); +BuyEvent(purchase_id: 6, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 7, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 7, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 7, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 8, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 8, item: "Soap", quantity: 1); + +Buyer(buyer_id: 11, purchase_id: 1); +Buyer(buyer_id: 12, purchase_id: 2); +Buyer(buyer_id: 13, purchase_id: 3); +Buyer(buyer_id: 14, purchase_id: 4); +Buyer(buyer_id: 12, purchase_id: 5); +Buyer(buyer_id: 13, purchase_id: 6); +Buyer(buyer_id: 14, purchase_id: 7); +Buyer(buyer_id: 11, purchase_id: 8); + +@OrderBy(Purchase, "purchase_id"); +Purchase(purchase_id:, items? List= x, buyer_id:) distinct :- + Buyer(buyer_id:, purchase_id:), + x = {item: item_name, quantity:, price:}, + BuyEvent(purchase_id:, item: item_name, quantity:), + Items(item: item_name, price:); + +@OrderBy(Test, "buyer_id"); +Test(buyer_id:, purchases:) :- + buyers Set= (b :- Buyer(buyer_id: b)), + buyer_id in buyers, + purchases List= ( + {purchase: items} :- + Purchase(items:, buyer_id:) + ); diff --git a/integration_tests/psql_purchase_test.l b/integration_tests/psql_purchase_test.l new file mode 100644 index 00000000..f0585e4c --- /dev/null +++ b/integration_tests/psql_purchase_test.l @@ -0,0 +1,69 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Items(item: "Soap", price: 20); +Items(item: "Milk", price: 10); +Items(item: "Bread", price: 5); +Items(item: "Coffee", price: 7); +Items(item: "Firewood", price: 15); + +MoreExpensiveThan(item1) List= item2 :- + Items(item: item1, price: price1), + Items(item: item2, price: price2), +price1 > price2; + +BuyEvent(purchase_id: 1, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 2, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 3, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 3, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 4, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 4, item: "Soap", quantity: 1); +BuyEvent(purchase_id: 5, item: "Milk", quantity: 4); +BuyEvent(purchase_id: 5, item: "Bread", quantity: 1); +BuyEvent(purchase_id: 5, item: "Coffee", quantity: 2); +BuyEvent(purchase_id: 6, item: "Firewood", quantity: 1); +BuyEvent(purchase_id: 6, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 7, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 7, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 7, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 8, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 8, item: "Soap", quantity: 1); + +Buyer(buyer_id: 11, purchase_id: 1); +Buyer(buyer_id: 12, purchase_id: 2); +Buyer(buyer_id: 13, purchase_id: 3); +Buyer(buyer_id: 14, purchase_id: 4); +Buyer(buyer_id: 12, purchase_id: 5); +Buyer(buyer_id: 13, purchase_id: 6); +Buyer(buyer_id: 14, purchase_id: 7); +Buyer(buyer_id: 11, purchase_id: 8); + +@OrderBy(Purchase, "purchase_id"); +Purchase(purchase_id:, items:, expensive_items:, buyer_id:) :- + Buyer(buyer_id:, purchase_id:), + items List= ( + {item:, quantity:, price:} :- + BuyEvent(purchase_id:, item:, quantity:), + Items(item:, price:) + ), + expensive_items List= ( + {item:, more_expensive_than: MoreExpensiveThan(item)} :- + item_record in items, + item = item_record.item + ); + +Test(..r) :- Purchase(..r); diff --git a/integration_tests/psql_purchase_test.txt b/integration_tests/psql_purchase_test.txt new file mode 100644 index 00000000..8eb80ded --- /dev/null +++ b/integration_tests/psql_purchase_test.txt @@ -0,0 +1,11 @@ ++----------------------------------------------+----------------------------------------------------------------------------------+---------- + 1 | {"(Soap,20,3)"} | {"(Soap,\"{Milk,Bread,Coffee,Firewood}\")"} | 11 + 2 | {"(Milk,10,1)"} | {"(Milk,\"{Bread,Coffee}\")"} | 12 + 3 | {"(Bread,5,2)","(Coffee,7,1)"} | {"(Coffee,{Bread})"} | 13 + 4 | {"(Soap,20,1)","(Firewood,15,5)"} | {"(Soap,\"{Milk,Bread,Coffee,Firewood}\")","(Firewood,\"{Milk,Bread,Coffee}\")"} | 14 + 5 | {"(Milk,10,4)","(Bread,5,1)","(Coffee,7,2)"} | {"(Milk,\"{Bread,Coffee}\")","(Coffee,{Bread})"} | 12 + 6 | {"(Soap,20,3)","(Firewood,15,1)"} | {"(Soap,\"{Milk,Bread,Coffee,Firewood}\")","(Firewood,\"{Milk,Bread,Coffee}\")"} | 13 + 7 | {"(Milk,10,1)","(Bread,5,2)","(Coffee,7,1)"} | {"(Milk,\"{Bread,Coffee}\")","(Coffee,{Bread})"} | 14 + 8 | {"(Soap,20,1)","(Firewood,15,5)"} | {"(Soap,\"{Milk,Bread,Coffee,Firewood}\")","(Firewood,\"{Milk,Bread,Coffee}\")"} | 11 +(8 rows) + diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 436fc5f3..f622f3f2 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -89,6 +89,8 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_purchase_test") + RunTest("psql_purchase2_test") RunTest("psql_combine_test") RunTest("psql_record_combine_test") RunTest("psql_structs_ground_test") diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 6367fba9..e35d490d 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -26,6 +26,10 @@ def TypesOfBultins(): reference_algebra.UnifyListElement(list_of_x, x) types_of_predicate = { + 'Aggr': { + 0: x, + 'logica_value': x + }, '=': { 'left': x, 'right': x, From 975727f1bc166605c09524febcb38727e1aa84ea Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 00:04:52 -0400 Subject: [PATCH 075/141] Remove premature internal variables ellimination. --- integration_tests/a.l | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 integration_tests/a.l diff --git a/integration_tests/a.l b/integration_tests/a.l new file mode 100644 index 00000000..6f5a434d --- /dev/null +++ b/integration_tests/a.l @@ -0,0 +1,32 @@ +Items(item: "Soap", price: 20); +Items(item: "Milk", price: 10); +Items(item: "Bread", price: 5); +Items(item: "Coffee", price: 7); +Items(item: "Firewood", price: 15); + +BuyEvent(purchase_id: 1, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 2, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 3, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 3, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 4, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 4, item: "Soap", quantity: 1); +BuyEvent(purchase_id: 5, item: "Milk", quantity: 4); +BuyEvent(purchase_id: 5, item: "Bread", quantity: 1); +BuyEvent(purchase_id: 5, item: "Coffee", quantity: 2); +BuyEvent(purchase_id: 6, item: "Firewood", quantity: 1); +BuyEvent(purchase_id: 6, item: "Soap", quantity: 3); +BuyEvent(purchase_id: 7, item: "Milk", quantity: 1); +BuyEvent(purchase_id: 7, item: "Bread", quantity: 2); +BuyEvent(purchase_id: 7, item: "Coffee", quantity: 1); +BuyEvent(purchase_id: 8, item: "Firewood", quantity: 5); +BuyEvent(purchase_id: 8, item: "Soap", quantity: 1); + + +Buyer(buyer_id: 11, purchase_id: 1); +Buyer(buyer_id: 12, purchase_id: 2); +Buyer(buyer_id: 13, purchase_id: 3); +Buyer(buyer_id: 14, purchase_id: 4); +Buyer(buyer_id: 12, purchase_id: 5); +Buyer(buyer_id: 13, purchase_id: 6); +Buyer(buyer_id: 14, purchase_id: 7); +Buyer(buyer_id: 11, purchase_id: 8); From c66cd98222a98f1a92f0bb249dd57b3bfedc8194 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 00:08:28 -0400 Subject: [PATCH 076/141] Remove premature internal variables ellimination. --- compiler/universe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/universe.py b/compiler/universe.py index bde8e644..f8894464 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -943,7 +943,8 @@ def SingleRuleSql(self, rule, s = rule_translate.ExtractRuleStructure( r, allocator, external_vocabulary) - s.ElliminateInternalVariables(assert_full_ellimination=False) + # TODO(2023 July): Was this always redundant? + # s.ElliminateInternalVariables(assert_full_ellimination=False) self.RunInjections(s, allocator) s.ElliminateInternalVariables(assert_full_ellimination=True) From bcc7b73ce8c3b04d1dab9e26e254c62f2ed93e2a Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 00:10:46 -0400 Subject: [PATCH 077/141] Delete a.l --- integration_tests/a.l | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 integration_tests/a.l diff --git a/integration_tests/a.l b/integration_tests/a.l deleted file mode 100644 index 6f5a434d..00000000 --- a/integration_tests/a.l +++ /dev/null @@ -1,32 +0,0 @@ -Items(item: "Soap", price: 20); -Items(item: "Milk", price: 10); -Items(item: "Bread", price: 5); -Items(item: "Coffee", price: 7); -Items(item: "Firewood", price: 15); - -BuyEvent(purchase_id: 1, item: "Soap", quantity: 3); -BuyEvent(purchase_id: 2, item: "Milk", quantity: 1); -BuyEvent(purchase_id: 3, item: "Bread", quantity: 2); -BuyEvent(purchase_id: 3, item: "Coffee", quantity: 1); -BuyEvent(purchase_id: 4, item: "Firewood", quantity: 5); -BuyEvent(purchase_id: 4, item: "Soap", quantity: 1); -BuyEvent(purchase_id: 5, item: "Milk", quantity: 4); -BuyEvent(purchase_id: 5, item: "Bread", quantity: 1); -BuyEvent(purchase_id: 5, item: "Coffee", quantity: 2); -BuyEvent(purchase_id: 6, item: "Firewood", quantity: 1); -BuyEvent(purchase_id: 6, item: "Soap", quantity: 3); -BuyEvent(purchase_id: 7, item: "Milk", quantity: 1); -BuyEvent(purchase_id: 7, item: "Bread", quantity: 2); -BuyEvent(purchase_id: 7, item: "Coffee", quantity: 1); -BuyEvent(purchase_id: 8, item: "Firewood", quantity: 5); -BuyEvent(purchase_id: 8, item: "Soap", quantity: 1); - - -Buyer(buyer_id: 11, purchase_id: 1); -Buyer(buyer_id: 12, purchase_id: 2); -Buyer(buyer_id: 13, purchase_id: 3); -Buyer(buyer_id: 14, purchase_id: 4); -Buyer(buyer_id: 12, purchase_id: 5); -Buyer(buyer_id: 13, purchase_id: 6); -Buyer(buyer_id: 14, purchase_id: 7); -Buyer(buyer_id: 11, purchase_id: 8); From 816d4543445055a42c9a0e59bb94492703545eb2 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Tue, 18 Jul 2023 04:13:25 +0000 Subject: [PATCH 078/141] Type inference unittesting. --- integration_tests/psql_purchase3_test.txt | 8 ++++++++ integration_tests/run_tests.py | 1 + 2 files changed, 9 insertions(+) create mode 100644 integration_tests/psql_purchase3_test.txt diff --git a/integration_tests/psql_purchase3_test.txt b/integration_tests/psql_purchase3_test.txt new file mode 100644 index 00000000..0397f5b8 --- /dev/null +++ b/integration_tests/psql_purchase3_test.txt @@ -0,0 +1,8 @@ + buyer_id | purchases +----------+-------------------------------------------------------------------------------------------------------------------------------- + 11 | {"(\"{\"\"(Soap,20,3)\"\"}\")","(\"{\"\"(Soap,20,1)\"\",\"\"(Firewood,15,5)\"\"}\")"} + 12 | {"(\"{\"\"(Milk,10,1)\"\"}\")","(\"{\"\"(Milk,10,4)\"\",\"\"(Bread,5,1)\"\",\"\"(Coffee,7,2)\"\"}\")"} + 13 | {"(\"{\"\"(Bread,5,2)\"\",\"\"(Coffee,7,1)\"\"}\")","(\"{\"\"(Soap,20,3)\"\",\"\"(Firewood,15,1)\"\"}\")"} + 14 | {"(\"{\"\"(Soap,20,1)\"\",\"\"(Firewood,15,5)\"\"}\")","(\"{\"\"(Milk,10,1)\"\",\"\"(Bread,5,2)\"\",\"\"(Coffee,7,1)\"\"}\")"} +(4 rows) + diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index f622f3f2..74e6d925 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -91,6 +91,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("psql_purchase_test") RunTest("psql_purchase2_test") + RunTest("psql_purchase3_test") RunTest("psql_combine_test") RunTest("psql_record_combine_test") RunTest("psql_structs_ground_test") From 9b49a2e8067c920434c313aae602789922ee5ba3 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Tue, 18 Jul 2023 04:37:13 +0000 Subject: [PATCH 079/141] Type inference unittesting. --- integration_tests/psql_combine2_test.l | 21 +++++++++++++++++++++ integration_tests/psql_combine2_test.txt | 6 ++++++ integration_tests/run_tests.py | 1 + 3 files changed, 28 insertions(+) create mode 100644 integration_tests/psql_combine2_test.l create mode 100644 integration_tests/psql_combine2_test.txt diff --git a/integration_tests/psql_combine2_test.l b/integration_tests/psql_combine2_test.l new file mode 100644 index 00000000..38bf76cb --- /dev/null +++ b/integration_tests/psql_combine2_test.l @@ -0,0 +1,21 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +T({t: 1, r: 3}); +T({t: 2, r: 4}); + +Test({a:1, dr: 2 * r}) = {b:2, c: d} :- d List= ({z: 2*i + t} :- i in Range(3)), T({t:, r:}); diff --git a/integration_tests/psql_combine2_test.txt b/integration_tests/psql_combine2_test.txt new file mode 100644 index 00000000..09bbf61b --- /dev/null +++ b/integration_tests/psql_combine2_test.txt @@ -0,0 +1,6 @@ + col0 | logica_value +-------+--------------------- + (1,6) | (2,"{(1),(3),(5)}") + (1,8) | (2,"{(2),(4),(6)}") +(2 rows) + diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 74e6d925..f8ec2220 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -93,6 +93,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("psql_purchase2_test") RunTest("psql_purchase3_test") RunTest("psql_combine_test") + RunTest("psql_combine2_test") RunTest("psql_record_combine_test") RunTest("psql_structs_ground_test") RunTest("psql_simple_structs_test") From 186b9c964190bcc71f90856f0ae966be30e9f1b8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:44:58 -0400 Subject: [PATCH 080/141] Support ascii graph in Jupyter. --- colab_logica.py | 41 +++++- common/concertina_lib.py | 35 ++++- compiler/dialect_libraries/psql_library.py | 2 + examples/more/Simple_Postgres.ipynb | 138 +++++++++++++++++++ logica.py | 15 +- tools/run_in_terminal.py | 26 +++- type_inference/research/types_of_builtins.py | 4 + 7 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 examples/more/Simple_Postgres.ipynb diff --git a/colab_logica.py b/colab_logica.py index 24f86a49..cb23c85f 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -16,6 +16,9 @@ """Library for using Logica in CoLab.""" +import getpass +import json + from .common import color from .common import concertina_lib @@ -23,6 +26,8 @@ from .compiler import rule_translate from .compiler import universe +from .type_inference.research import infer + import IPython from IPython.core.magic import register_cell_magic @@ -70,6 +75,8 @@ PREAMBLE = None +DISPLAY_MODE = 'colab' # or colab-text + def SetPreamble(preamble): global PREAMBLE @@ -83,6 +90,21 @@ def SetDbConnection(connection): global DB_CONNECTION DB_CONNECTION = connection +def ConnectToPostgres(mode='interactive'): + import psycopg2 + if mode == 'interactive': + print('Please enter PostgreSQL connection config in JSON format.') + print('Example:') + print('{"host": "myhost", "database": "megadb", ' + '"user": "myuser", "password": "42"}') + connection_str = getpass.getpass() + elif mode == 'environment': + connection_str = os.environ.get('LOGICA_PSQL_CONNECTION') + else: + assert False, 'Unknown mode:' + mode + connection_json = json.loads(connection_str) + SetDbConnection(psycopg2.connect(**connection_json)) + def EnsureAuthenticatedUser(): global USER_AUTHENTICATED global PROJECT @@ -142,12 +164,17 @@ def RunSQL(sql, engine, connection=None, is_final=False): client = bigquery.Client(project=PROJECT) return client.query(sql).to_dataframe() elif engine == 'psql': - # Sorry, this is not looking good. - from sqlalchemy import text if is_final: - return pandas.read_sql(text(sql), connection) + cursor = connection.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + df = pandas.DataFrame( + rows, columns=[d[0] for d in cursor.description]) + return df else: - return connection.execute(text(sql)) + cursor = connection.cursor() + cursor.execute(sql) + connection.commit() elif engine == 'sqlite': try: if is_final: @@ -215,6 +242,9 @@ def Logica(line, cell, run_query): except rule_translate.RuleCompileException as e: e.ShowMessage() return + except infer.TypeErrorCaughtException as e: + e.ShowMessage() + return engine = program.annotations.Engine() @@ -273,7 +303,8 @@ def Logica(line, cell, run_query): 'for now.') result_map = concertina_lib.ExecuteLogicaProgram( - executions, sql_runner=sql_runner, sql_engine=engine) + executions, sql_runner=sql_runner, sql_engine=engine, + display_mode=DISPLAY_MODE) for idx, predicate in enumerate(predicates): t = result_map[predicate] diff --git a/common/concertina_lib.py b/common/concertina_lib.py index 94a96741..54f5d1fc 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -4,10 +4,15 @@ try: import graphviz +except: + print('Could not import graphviz tools in Concertina.') + +try: + from IPython.display import HTML from IPython.display import display from IPython.display import update_display except: - print('Could not import CoLab tools in Concertina.') + print('Could not import IPython in Concertina.') if '.' not in __package__: from common import graph_art @@ -75,7 +80,7 @@ def __init__(self, config, engine, display_mode='colab'): self.all_actions = {a["name"] for a in self.config} self.complete_actions = set() self.running_actions = set() - assert display_mode in ('colab', 'terminal'), ( + assert display_mode in ('colab', 'terminal', 'colab-text'), ( 'Unrecognized display mode: %s' % display_mode) self.display_mode = display_mode self.display_id = self.GetDisplayId() @@ -137,7 +142,14 @@ def AsNodesAndEdges(self): """Nodes and edges to display in terminal.""" def ColoredNode(node): if node in self.running_actions: - return '\033[1m\033[93m' + node + '\033[0m' + if self.display_mode == 'terminal': + return '\033[1m\033[93m' + node + '\033[0m' + elif self.display_mode == 'colab-text': + return ( + '' + node + '' + ) + else: + assert False, self.display_mode else: return node nodes = [] @@ -150,11 +162,24 @@ def ColoredNode(node): edges.append([prerequisite_node, a_node]) return nodes, edges + def StateAsSimpleHTML(self): + style = ';'.join([ + 'border: 1px solid rgba(0, 0, 0, 0.3)', + 'width: fit-content;', + 'padding: 20px', + 'border-radius: 5px', + 'box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.2)']) + return HTML('
%s
' % ( + style, self.AsTextPicture(updating=False))) + def Display(self): if self.display_mode == 'colab': display(self.AsGraphViz(), display_id=self.display_id) elif self.display_mode == 'terminal': print(self.AsTextPicture(updating=False)) + elif self.display_mode == 'colab-text': + display(self.StateAsSimpleHTML(), + display_id=self.display_id) else: assert 'Unexpected mode:', self.display_mode @@ -163,6 +188,10 @@ def UpdateDisplay(self): update_display(self.AsGraphViz(), display_id=self.display_id) elif self.display_mode == 'terminal': print(self.AsTextPicture(updating=True)) + elif self.display_mode == 'colab-text': + update_display( + self.StateAsSimpleHTML(), + display_id=self.display_id) else: assert 'Unexpected mode:', self.display_mode diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 2bd1e31b..0e80cfd3 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -37,4 +37,6 @@ "ARRAY_AGG({value} order by {arg})", {arg: a.arg, value: a.value}); +RecordAsJson(r) = SqlExpr( + "ROW_TO_JSON({r})", {r:}); """ diff --git a/examples/more/Simple_Postgres.ipynb b/examples/more/Simple_Postgres.ipynb new file mode 100644 index 00000000..4a96aec7 --- /dev/null +++ b/examples/more/Simple_Postgres.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "7e89b446", + "metadata": {}, + "source": [ + "# Example of running Postgres with text Pipeline overview" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58f2b0db", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from logica import colab_logica\n", + "colab_logica.ConnectToPostgres('interactive')\n", + "colab_logica.DISPLAY_MODE = 'colab-text'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "82c02eb6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Query is stored at \u001b[1mGreeting_sql\u001b[0m variable.\n" + ] + }, + { + "data": { + "text/html": [ + "
 â–š  Greeting
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running predicate: Greeting (0 seconds)\n", + "The following table is stored at \u001b[1mGreeting\u001b[0m variable.\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col0
0Hello world!
\n", + "
" + ], + "text/plain": [ + " col0\n", + "0 Hello world!" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " \n" + ] + } + ], + "source": [ + "%%logica Greeting\n", + "\n", + "@Engine(\"psql\");\n", + "\n", + "Greeting(\"Hello world!\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/logica.py b/logica.py index 46a508cc..604baaad 100755 --- a/logica.py +++ b/logica.py @@ -139,6 +139,7 @@ def main(argv): print('Not enough arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 + predicates = argv[3] if argv[1] == '-': filename = '/dev/stdin' @@ -158,6 +159,13 @@ def main(argv): if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 + + if command == 'run_in_terminal': + from tools import run_in_terminal + artistic_table = run_in_terminal.Run(filename, predicates) + print(artistic_table) + return + program_text = open(filename).read() try: @@ -187,8 +195,6 @@ def main(argv): type_error_checker.CheckForError() return 0 - predicates = argv[3] - user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') @@ -265,11 +271,6 @@ def main(argv): assert False, 'Unknown engine: %s' % engine print(o.decode()) - if command == 'run_in_terminal': - from tools import run_in_terminal - artistic_table = run_in_terminal.Run(filename, predicate) - print(artistic_table) - def run_main(): """Run main function with system arguments.""" diff --git a/tools/run_in_terminal.py b/tools/run_in_terminal.py index fe3ee985..8a20c8b2 100644 --- a/tools/run_in_terminal.py +++ b/tools/run_in_terminal.py @@ -16,6 +16,9 @@ # Utility to run pipeline in terminal with ASCII art showing progress. +import json +import os + if not __package__ or '.' not in __package__: from common import concertina_lib from compiler import universe @@ -31,7 +34,7 @@ class SqlRunner(object): def __init__(self, engine): self.engine = engine - assert engine in ['sqlite', 'bigquery'] + assert engine in ['sqlite', 'bigquery', 'psql'] if engine == 'sqlite': self.connection = sqlite3_logica.SqliteConnect() else: @@ -45,6 +48,16 @@ def __init__(self, engine): credentials, project = auth.default() else: credentials, project = None, None + if engine == 'psql': + import psycopg2 + if os.environ.get('LOGICA_PSQL_CONNECTION'): + connection_json = json.loads(os.environ.get('LOGICA_PSQL_CONNECTION')) + else: + assert False, ( + 'Please provide PSQL connection parameters ' + 'in LOGICA_PSQL_CONNECTION') + self.connection = psycopg2.connect(**connection_json) + self.bq_credentials = credentials self.bq_project = project @@ -68,10 +81,17 @@ def RunSQL(sql, engine, connection=None, is_final=False, elif engine == 'psql': import pandas if is_final: - df = pandas.read_sql(sql, connection) + cursor = connection.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + df = pandas.DataFrame( + rows, columns=[d[0] for d in cursor.description]) + connection.close() return list(df.columns), [list(r) for _, r in df.iterrows()] else: - return connection.execute(sql) + cursor = connection.cursor() + cursor.execute(sql) + connection.commit() elif engine == 'sqlite': try: if is_final: diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index e35d490d..88a6b123 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -127,6 +127,10 @@ def TypesOfBultins(): 'ValueOfUnnested': { 0: x, 'logica_value': x + }, + 'RecordAsJson': { + 0: reference_algebra.OpenRecord({}), + 'logica_value': 'Str' } } return { From 00c4234db6a7892ecb31189a86286bbbf9ac62a6 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:07:05 -0400 Subject: [PATCH 081/141] Type inference improvements. --- type_inference/research/types_of_builtins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 88a6b123..e01ecd7d 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -30,6 +30,11 @@ def TypesOfBultins(): 0: x, 'logica_value': x }, + '==': { + 'left': x, + 'right': x, + 'logica_value': 'Any' # TODO: Add Boolean. + }, '=': { 'left': x, 'right': x, From a63625ffe71d74fbc8879c0869ac313d872099e2 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:18:21 -0400 Subject: [PATCH 082/141] Type inference improvements. --- type_inference/research/infer.py | 14 ---------- type_inference/research/reference_algebra.py | 27 +++++++++++++++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index a2d8132a..435f8f8d 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -68,20 +68,6 @@ def NiceMessage(self): def HelpfulErrorMessage(self): result = str(self.type_error) - if (isinstance(self.type_error[0], dict) and - isinstance(self.type_error[1], dict)): - if isinstance(self.type_error[0], - reference_algebra.ClosedRecord): - a, b = self.type_error - else: - b, a = self.type_error - if (isinstance(a, reference_algebra.ClosedRecord) and - isinstance(b, reference_algebra.OpenRecord) and - list(b)[0] not in a.keys() ): - result = ( - 'record ' + str(a) + ' does not have field ' + list(b)[0] + '.' - ) - if self.refers_to_variable: result = color.Format( 'Variable {warning}%s{end} ' % diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 80d9d52e..d6fff558 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -39,9 +39,30 @@ def __repr__(self): class BadType(tuple): def __str__(self): - colored_t1 = color.Format('{warning}{t}{end}', args_dict={'t': RenderType(self[0])}) - colored_t2 = color.Format('{warning}{t}{end}', args_dict={'t': RenderType(self[1])}) - + if (isinstance(self[0], dict) and + isinstance(self[1], dict)): + if isinstance(self[0], + ClosedRecord): + a, b = self + else: + b, a = self + else: + a, b = self + + colored_t1 = color.Format('{warning}{t}{end}', + args_dict={'t': RenderType(a)}) + colored_t2 = color.Format('{warning}{t}{end}', + args_dict={'t': RenderType(b)}) + + if (isinstance(a, ClosedRecord) and + isinstance(b, OpenRecord) and + list(b)[0] not in a.keys()): + colored_e = color.Format( + '{warning}{t}{end}', args_dict={'t': RenderType(list(b)[0])}) + return ( + f'is a record {colored_t1} and it does not have ' + + f'field {colored_e}, which is addressed.' + ) return ( f'is implied to be {colored_t1} and ' + f'simultaneously {colored_t2}, which is impossible.') From 1b2c6bc20217bf446afb04c7c8eb35b15647d5aa Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:24:57 -0400 Subject: [PATCH 083/141] Type inference improvements. --- type_inference/research/types_of_builtins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index e01ecd7d..af6d0216 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -136,8 +136,13 @@ def TypesOfBultins(): 'RecordAsJson': { 0: reference_algebra.OpenRecord({}), 'logica_value': 'Str' + }, + '>': { + 'left': x, + 'right': x } } + types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] return { p: {k: reference_algebra.TypeReference(v) for k, v in types.items()} From 1a37ea0df6149fa17d3640869efc1612b4fbf135 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:42:36 -0400 Subject: [PATCH 084/141] Type inference improvements. --- compiler/dialects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index 3e5928d2..fe36eda8 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -137,7 +137,8 @@ def BuiltInFunctions(self): 'Element': '({0})[{1} + 1]', 'Size': 'ARRAY_LENGTH(%s, 1)', 'Count': 'COUNT(DISTINCT {0})', - 'MagicalEntangle': '(CASE WHEN {1} = 0 THEN {0} ELSE NULL END)' + 'MagicalEntangle': '(CASE WHEN {1} = 0 THEN {0} ELSE NULL END)', + 'ArrayConcat': '{0} || {1}' } def InfixOperators(self): From f7c2d0a31feb48a90c3b96e8ac22dfcd6abba573 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:52:56 -0400 Subject: [PATCH 085/141] Type inference improvements. --- type_inference/research/types_of_builtins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index af6d0216..71cb6ead 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -140,6 +140,11 @@ def TypesOfBultins(): '>': { 'left': x, 'right': x + }, + 'ArrayConcat': { + 0: [x], + 1: [x], + 'logica_value': [x] } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] From ad7b2dea87e89aa49608217d70adb661ee21d159 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 01:08:53 -0400 Subject: [PATCH 086/141] Type inference improvements. --- type_inference/research/infer.py | 2 +- .../research/integration_tests/run_tests.py | 1 + .../typing_palindrome_puzzle_test.l | 33 + .../typing_palindrome_puzzle_test.txt | 1910 +++++++++++++++++ type_inference/research/types_of_builtins.py | 9 +- 5 files changed, 1953 insertions(+), 2 deletions(-) create mode 100644 type_inference/research/integration_tests/typing_palindrome_puzzle_test.l create mode 100644 type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 435f8f8d..c65c23de 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -214,7 +214,7 @@ def ExtractPredicateName(node): result = {} for rule in rules: p, ds = ExtractDendencies(rule) - result[p] = list(sorted(set(ds) - set([p]))) + result[p] = list(set(sorted(set(ds) - set([p]))) | set(result.get(p, []))) return result def BuildComplexities(dependencies): diff --git a/type_inference/research/integration_tests/run_tests.py b/type_inference/research/integration_tests/run_tests.py index fd209807..81a04057 100644 --- a/type_inference/research/integration_tests/run_tests.py +++ b/type_inference/research/integration_tests/run_tests.py @@ -26,6 +26,7 @@ def RunTypesTest(name, src=None, golden=None): logica_test.TestManager.RunTypesTest(name, src, golden) def RunAll(): + RunTypesTest('typing_palindrome_puzzle_test') RunTypesTest('typing_kitchensync_test') RunTypesTest('typing_basic_test') RunTypesTest('typing_aggregation_test') diff --git a/type_inference/research/integration_tests/typing_palindrome_puzzle_test.l b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.l new file mode 100644 index 00000000..6284bc63 --- /dev/null +++ b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.l @@ -0,0 +1,33 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Word("abbacdcaaaxx"); + +Char(i) = Substr(word, i + 1, 1) :- Word(word), i in Range(Length(word)); + +@Ground(Palindrome); +Palindrome(i, i) = Char(i); +Palindrome(i, i + 1) = Char(i) ++ Char(i + 1) :- Char(i) == Char(i + 1); +Palindrome(i - 1, j + 1) = Char(i - 1) ++ Palindrome(i, j) ++ Char(j + 1) :- Palindrome(i, j), Char(i - 1) == Char(j + 1); + +@Ground(Path); +Path(i, j) = {path: [Palindrome(i, j)]} distinct; +Path(i, k) = {path: ArrayConcat(Path(i, j).path, Path(j + 1, k).path)} distinct; + +ShortestPath() ArgMin= path -> Size(path.path) :- path == Path(); + diff --git a/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt new file mode 100644 index 00000000..c349d584 --- /dev/null +++ b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt @@ -0,0 +1,1910 @@ +[ + { + "full_text": "@Engine(\"psql\")", + "head": { + "predicate_name": "@Engine", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "\"psql\"", + "literal": { + "the_string": { + "the_string": "psql" + } + } + } + } + } + ] + } + } + }, + { + "full_text": "Word(\"abbacdcaaaxx\")", + "head": { + "predicate_name": "Word", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "\"abbacdcaaaxx\"", + "literal": { + "the_string": { + "the_string": "abbacdcaaaxx" + } + }, + "type": { + "the_type": "Str", + "type_id": 0 + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "Word", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "word", + "type": { + "the_type": "Str", + "type_id": 1 + }, + "variable": { + "var_name": "word" + } + } + } + } + ] + } + } + }, + { + "inclusion": { + "element": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + }, + "list": { + "call": { + "predicate_name": "Range", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "Length", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "word", + "type": { + "the_type": "Str", + "type_id": 1 + }, + "variable": { + "var_name": "word" + } + } + } + } + ] + } + }, + "expression_heritage": "Length(word)", + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + } + ] + } + }, + "expression_heritage": "Range(Length(word))", + "type": { + "element_type_name": "numeric", + "the_type": [ + "Num" + ], + "type_id": 6 + } + } + } + } + ] + } + }, + "full_text": "Char(i) = Substr(word, i + 1, 1) :- Word(word), i in Range(Length(word))", + "head": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "call": { + "predicate_name": "Substr", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "word", + "type": { + "the_type": "Str", + "type_id": 1 + }, + "variable": { + "var_name": "word" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 4 + } + } + } + } + ] + } + }, + "expression_heritage": "i + 1", + "type": { + "the_type": "Num", + "type_id": 3 + } + } + } + }, + { + "field": 2, + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + ] + } + }, + "expression_heritage": "Substr(word, i + 1, 1)", + "type": { + "the_type": "Str", + "type_id": 2 + } + } + } + } + ] + } + } + }, + { + "full_text": "@Ground(Palindrome)", + "head": { + "predicate_name": "@Ground", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "Palindrome", + "literal": { + "the_predicate": { + "predicate_name": "Palindrome" + } + } + } + } + } + ] + } + } + }, + { + "full_text": "Palindrome(i, i) = Char(i)", + "head": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i)", + "type": { + "the_type": "Str", + "type_id": 1 + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "unification": { + "left_hand_side": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i)", + "type": { + "the_type": "Str", + "type_id": 8 + } + }, + "right_hand_side": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 11 + } + } + } + } + ] + } + }, + "expression_heritage": "i + 1", + "type": { + "the_type": "Num", + "type_id": 10 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i + 1)", + "type": { + "the_type": "Str", + "type_id": 9 + } + } + } + } + ] + } + }, + "full_text": "Palindrome(i, i + 1) = Char(i) ++ Char(i + 1) :- Char(i) == Char(i + 1)", + "head": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 2 + } + } + } + } + ] + } + }, + "expression_heritage": "i + 1", + "type": { + "the_type": "Num", + "type_id": 1 + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "call": { + "predicate_name": "++", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i)", + "type": { + "the_type": "Str", + "type_id": 4 + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 7 + } + } + } + } + ] + } + }, + "expression_heritage": "i + 1", + "type": { + "the_type": "Num", + "type_id": 6 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i + 1)", + "type": { + "the_type": "Str", + "type_id": 5 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i) ++ Char(i + 1)", + "type": { + "the_type": "Str", + "type_id": 3 + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + } + ] + } + } + }, + { + "unification": { + "left_hand_side": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "-", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 18 + } + } + } + } + ] + } + }, + "expression_heritage": "i - 1", + "type": { + "the_type": "Num", + "type_id": 17 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i - 1)", + "type": { + "the_type": "Str", + "type_id": 15 + } + }, + "right_hand_side": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 20 + } + } + } + } + ] + } + }, + "expression_heritage": "j + 1", + "type": { + "the_type": "Num", + "type_id": 19 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(j + 1)", + "type": { + "the_type": "Str", + "type_id": 16 + } + } + } + } + ] + } + }, + "full_text": "Palindrome(i - 1, j + 1) = Char(i - 1) ++ Palindrome(i, j) ++ Char(j + 1) :- Palindrome(i, j), Char(i - 1) == Char(j + 1)", + "head": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "-", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 3 + } + } + } + } + ] + } + }, + "expression_heritage": "i - 1", + "type": { + "the_type": "Num", + "type_id": 2 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 5 + } + } + } + } + ] + } + }, + "expression_heritage": "j + 1", + "type": { + "the_type": "Num", + "type_id": 4 + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "call": { + "predicate_name": "++", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "call": { + "predicate_name": "++", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "-", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 10 + } + } + } + } + ] + } + }, + "expression_heritage": "i - 1", + "type": { + "the_type": "Num", + "type_id": 9 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i - 1)", + "type": { + "the_type": "Str", + "type_id": 8 + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "call": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + } + ] + } + }, + "expression_heritage": "Palindrome(i, j)", + "type": { + "the_type": "Str", + "type_id": 11 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i - 1) ++ Palindrome(i, j)", + "type": { + "the_type": "Str", + "type_id": 7 + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "call": { + "predicate_name": "Char", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 14 + } + } + } + } + ] + } + }, + "expression_heritage": "j + 1", + "type": { + "the_type": "Num", + "type_id": 13 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(j + 1)", + "type": { + "the_type": "Str", + "type_id": 12 + } + } + } + } + ] + } + }, + "expression_heritage": "Char(i - 1) ++ Palindrome(i, j) ++ Char(j + 1)", + "type": { + "the_type": "Str", + "type_id": 6 + } + } + } + } + ] + } + } + }, + { + "full_text": "@Ground(Path)", + "head": { + "predicate_name": "@Ground", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "Path", + "literal": { + "the_predicate": { + "predicate_name": "Path" + } + } + } + } + } + ] + } + } + }, + { + "full_text": "Path(i, j) = {path: [Palindrome(i, j)]} distinct", + "head": { + "predicate_name": "Path_MultBodyAggAux", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "expression_heritage": "{path: [Palindrome(i, j)]}", + "record": { + "field_value": [ + { + "field": "path", + "value": { + "expression": { + "expression_heritage": "[Palindrome(i, j)]", + "literal": { + "the_list": { + "element": [ + { + "call": { + "predicate_name": "Palindrome", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "j" + } + } + } + } + ] + } + }, + "expression_heritage": "Palindrome(i, j)", + "type": { + "the_type": "Str", + "type_id": 4 + } + } + ] + } + }, + "type": { + "element_type_name": "text", + "the_type": [ + "Str" + ], + "type_id": 3 + } + } + } + } + ] + }, + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 2, + "type_name": "logicarecord808144452" + } + } + } + } + ] + } + } + }, + { + "full_text": "Path(i, k) = {path: ArrayConcat(Path(i, j).path, Path(j + 1, k).path)} distinct", + "head": { + "predicate_name": "Path_MultBodyAggAux", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "k", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "k" + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "expression_heritage": "{path: ArrayConcat(Path(i, j).path, Path(j + 1, k).path)}", + "record": { + "field_value": [ + { + "field": "path", + "value": { + "expression": { + "call": { + "predicate_name": "ArrayConcat", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "Path(i, j).path", + "subscript": { + "record": { + "call": { + "predicate_name": "Path", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "i", + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": "i" + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "j" + } + } + } + } + ] + } + }, + "expression_heritage": "Path(i, j)", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 6, + "type_name": "logicarecord808144452" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "path" + } + } + } + }, + "type": { + "element_type_name": "text", + "the_type": [ + "Str" + ], + "type_id": 5 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "Path(j + 1, k).path", + "subscript": { + "record": { + "call": { + "predicate_name": "Path", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "+", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "j", + "type": { + "the_type": "Num", + "type_id": 2 + }, + "variable": { + "var_name": "j" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "expression_heritage": "1", + "literal": { + "the_number": { + "number": "1" + } + }, + "type": { + "the_type": "Num", + "type_id": 10 + } + } + } + } + ] + } + }, + "expression_heritage": "j + 1", + "type": { + "the_type": "Num", + "type_id": 9 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "expression_heritage": "k", + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": "k" + } + } + } + } + ] + } + }, + "expression_heritage": "Path(j + 1, k)", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 8, + "type_name": "logicarecord808144452" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "path" + } + } + } + }, + "type": { + "element_type_name": "text", + "the_type": [ + "Str" + ], + "type_id": 7 + } + } + } + } + ] + } + }, + "expression_heritage": "ArrayConcat(Path(i, j).path, Path(j + 1, k).path)", + "type": { + "element_type_name": "text", + "the_type": [ + "Str" + ], + "type_id": 4 + } + } + } + } + ] + }, + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 3, + "type_name": "logicarecord808144452" + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "unification": { + "left_hand_side": { + "expression_heritage": "path", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 0, + "type_name": "logicarecord808144452" + }, + "variable": { + "var_name": "path" + } + }, + "right_hand_side": { + "call": { + "predicate_name": "Path", + "record": { + "field_value": [] + } + }, + "expression_heritage": "Path()", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 5, + "type_name": "logicarecord808144452" + } + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "ShortestPath() ArgMin= path -> Size(path.path) :- path == Path()", + "head": { + "predicate_name": "ShortestPath", + "record": { + "field_value": [ + { + "field": "logica_value", + "value": { + "aggregation": { + "expression": { + "call": { + "predicate_name": "ArgMin", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "call": { + "predicate_name": "->", + "record": { + "field_value": [ + { + "field": "left", + "value": { + "expression": { + "expression_heritage": "path", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 0, + "type_name": "logicarecord808144452" + }, + "variable": { + "var_name": "path" + } + } + } + }, + { + "field": "right", + "value": { + "expression": { + "call": { + "predicate_name": "Size", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "expression_heritage": "path.path", + "subscript": { + "record": { + "expression_heritage": "path", + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 0, + "type_name": "logicarecord808144452" + }, + "variable": { + "var_name": "path" + } + }, + "subscript": { + "literal": { + "the_symbol": { + "symbol": "path" + } + } + } + }, + "type": { + "element_type_name": "text", + "the_type": [ + "Str" + ], + "type_id": 4 + } + } + } + } + ] + } + }, + "expression_heritage": "Size(path.path)", + "type": { + "the_type": "Num", + "type_id": 3 + } + } + } + } + ] + } + }, + "expression_heritage": "path -> Size(path.path)", + "type": { + "the_type": { + "arg": { + "path": [ + "Str" + ] + }, + "value": "Num" + }, + "type_id": 2, + "type_name": "logicarecord659526184" + } + } + } + } + ] + } + }, + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 1, + "type_name": "logicarecord808144452" + } + } + } + } + } + ] + } + } + }, + { + "body": { + "conjunction": { + "conjunct": [ + { + "predicate": { + "predicate_name": "Path_MultBodyAggAux", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": 0 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": 1 + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 2, + "type_name": "logicarecord808144452" + }, + "variable": { + "var_name": "logica_value" + } + } + } + } + ] + } + } + } + ] + } + }, + "distinct_denoted": true, + "full_text": "Path(i, k) = {path: ArrayConcat(Path(i, j).path, Path(j + 1, k).path)} distinct", + "head": { + "predicate_name": "Path", + "record": { + "field_value": [ + { + "field": 0, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 0 + }, + "variable": { + "var_name": 0 + } + } + } + }, + { + "field": 1, + "value": { + "expression": { + "type": { + "the_type": "Num", + "type_id": 1 + }, + "variable": { + "var_name": 1 + } + } + } + }, + { + "field": "logica_value", + "value": { + "expression": { + "type": { + "the_type": { + "path": [ + "Str" + ] + }, + "type_id": 2, + "type_name": "logicarecord808144452" + }, + "variable": { + "var_name": "logica_value" + } + } + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 71cb6ead..5ee14b5f 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -139,12 +139,19 @@ def TypesOfBultins(): }, '>': { 'left': x, - 'right': x + 'right': x, + 'logica_value': 'Any' }, 'ArrayConcat': { 0: [x], 1: [x], 'logica_value': [x] + }, + 'Substr': { + 0: 'Str', + 1: 'Num', + 2: 'Num', + 'logica_value': 'Str' } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] From c64a9b0421782ad427de0d2fec43973e304ece3b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:12:11 -0400 Subject: [PATCH 087/141] Type inference: guarding against lists of lists. --- .../integration_tests/typing_basic_test.txt | 6 +-- type_inference/research/reference_algebra.py | 39 +++++++++++++++---- type_inference/research/types_of_builtins.py | 37 +++++++++--------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index a0b4f8ed..38a94931 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -110,7 +110,7 @@ "expression_heritage": "l", "type": { "the_type": [ - "Any" + "Singular" ], "type_id": 3 }, @@ -129,7 +129,7 @@ "element": { "expression_heritage": "e", "type": { - "the_type": "Any", + "the_type": "Singular", "type_id": 4 }, "variable": { @@ -140,7 +140,7 @@ "expression_heritage": "l", "type": { "the_type": [ - "Any" + "Singular" ], "type_id": 3 }, diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index d6fff558..e192b08d 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -46,6 +46,8 @@ def __str__(self): a, b = self else: b, a = self + elif self[1] == 'Singular': + b, a = self else: a, b = self @@ -63,6 +65,17 @@ def __str__(self): f'is a record {colored_t1} and it does not have ' + f'field {colored_e}, which is addressed.' ) + if a == 'Singular': + assert isinstance(b, list), 'Fatally incorrect singular error: %s' % b + return ( + f'belongs to a list, but is implied to be {colored_t2}. ' + f'Logica has to follow existing DB practice (Posgres, BigQuery) ' + f'and disallow lists to be elements of lists. This includes ' + f'ArgMax and ArgMin aggregations, as they use SQL arrays as an ' + f'intermediate. Kindly wrap your inner list into a single field ' + f'record.' + ) + return ( f'is implied to be {colored_t1} and ' + f'simultaneously {colored_t2}, which is impossible.') @@ -156,6 +169,8 @@ def VeryConcreteType(t, upward=None): def IsFullyDefined(t): if t == 'Any': return False + if t == 'Singular': + return False if isinstance(t, str): return True if isinstance(t, BadType): @@ -175,16 +190,18 @@ def Rank(x): return -1 if x == 'Any': return 0 - if x == 'Num': + if x == 'Singular': return 1 - if x == 'Str': + if x == 'Num': return 2 - if isinstance(x, list): + if x == 'Str': return 3 - if isinstance(x, OpenRecord): + if isinstance(x, list): return 4 - if isinstance(x, ClosedRecord): + if isinstance(x, OpenRecord): return 5 + if isinstance(x, ClosedRecord): + return 6 assert False, 'Bad type: %s' % x @@ -221,6 +238,15 @@ def Unify(a, b): if concrete_a == 'Any': a.target = b return + + if concrete_a == 'Singular': + if isinstance(concrete_b, list): + a.target, b.target = ( + Incompatible(a.target, b.target), + Incompatible(b.target, a.target)) + return + a.target = b + return if concrete_a in ('Num', 'Str'): if concrete_a == concrete_b: @@ -299,11 +325,10 @@ def UnifyFriendlyRecords(a, b, record_type): def UnifyListElement(a_list, b_element): """Analysis of expression `b in a`.""" + Unify(b_element, TypeReference.To('Singular')) b = TypeReference([b_element]) Unify(a_list, b) - # TODO: Prohibit lists of lists. - def UnifyRecordField(a_record, field_name, b_field_value): """Analysis of expresson `a.f = b`.""" diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 5ee14b5f..f2f18355 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -22,8 +22,9 @@ def TypesOfBultins(): x = reference_algebra.TypeReference('Any') y = reference_algebra.TypeReference('Any') - list_of_x = reference_algebra.TypeReference('Any') - reference_algebra.UnifyListElement(list_of_x, x) + list_of_e = reference_algebra.TypeReference('Any') + e = reference_algebra.TypeReference('Singular') + reference_algebra.UnifyListElement(list_of_e, e) types_of_predicate = { 'Aggr': { @@ -73,8 +74,8 @@ def TypesOfBultins(): 'logica_value': 'Num' }, 'List': { - 0: x, - 'logica_value': list_of_x + 0: e, + 'logica_value': list_of_e }, '->': { 'left': x, @@ -82,22 +83,22 @@ def TypesOfBultins(): 'logica_value': reference_algebra.ClosedRecord({'arg': x, 'value': y}) }, 'ArgMin': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), - 'logica_value': x + 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), + 'logica_value': e }, 'ArgMax': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), - 'logica_value': x + 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), + 'logica_value': e }, 'ArgMinK': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), 1: 'Num', - 'logica_value': [x] + 'logica_value': [e] }, 'ArgMaxK': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), 1: 'Num', - 'logica_value': [x] + 'logica_value': [e] }, 'Range': { 0: 'Num', @@ -108,7 +109,7 @@ def TypesOfBultins(): 'logica_value': 'Num' }, 'Size': { - 0: ['Any'], + 0: ['Singular'], 'logica_value': 'Num' }, '-': { @@ -126,8 +127,8 @@ def TypesOfBultins(): 'logica_value': y }, 'Array': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), - 'logica_value': y + 0: reference_algebra.ClosedRecord({'arg': x, 'value': e}), + 'logica_value': e }, 'ValueOfUnnested': { 0: x, @@ -143,9 +144,9 @@ def TypesOfBultins(): 'logica_value': 'Any' }, 'ArrayConcat': { - 0: [x], - 1: [x], - 'logica_value': [x] + 0: [e], + 1: [e], + 'logica_value': [e] }, 'Substr': { 0: 'Str', From 848aeaaf0fb7343eb7ac572d3514d4d73fb7dd21 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 16:31:50 -0400 Subject: [PATCH 088/141] Type inference: predicate arguments handling. --- type_inference/research/infer.py | 88 +++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index c65c23de..aef2ed5d 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -57,13 +57,20 @@ def ReplaceIfMoreUseful(self, type_error, context_string, refers_to_variable, re else: pass - def NiceMessage(self): + @classmethod + def BuildNiceMessage(self, context_string, error_message): result_lines = [ color.Format('{underline}Type analysis:{end}'), - self.context_string, '', - color.Format('[ {error}Error{end} ] ') + self.HelpfulErrorMessage()] - + context_string, '', + color.Format('[ {error}Error{end} ] ') + error_message] return '\n'.join(result_lines) + + def NiceMessage(self): + if (isinstance(self.type_error[0], str) and + self.type_error[0].startswith('VERBATIM:')): + return self.type_error[0].removeprefix('VERBATIM:') + return self.BuildNiceMessage(self.context_string, + self.HelpfulErrorMessage()) def HelpfulErrorMessage(self): result = str(self.type_error) @@ -265,37 +272,90 @@ def MindPodLiterals(self): Walk(self.rule, ActMindingPodLiterals) def ActMindingBuiltinFieldTypes(self, node): - def InstillTypes(field_value, signature, output_value_type): + def InstillTypes(predicate_name, + field_value, signature, output_value): copier = reference_algebra.TypeStructureCopier() copy = copier.CopyConcreteOrReferenceType - if output_value_type: - reference_algebra.Unify( - output_value_type, - copy(signature['logica_value'])) + if output_value: + output_value_type = output_value['type']['the_type'] + if 'logica_value' in signature: + reference_algebra.Unify( + output_value_type, + copy(signature['logica_value'])) + else: + error_message = ( + ContextualizedError.BuildNiceMessage( + output_value['expression_heritage'].Display(), + 'Predicate %s is not a function, but was called as such.' % + color.Format('{warning}%s{end}') % predicate_name, + ) + ) + error = reference_algebra.BadType( + ('VERBATIM:' + error_message, + output_value_type.target)) + output_value_type.target = reference_algebra.TypeReference.To(error) + for fv in field_value: if fv['field'] in signature: reference_algebra.Unify( fv['value']['expression']['type']['the_type'], copy(signature[fv['field']])) + elif fv['field'] == '*': + args = copy(reference_algebra.ClosedRecord(signature)) + reference_algebra.Unify( + fv['value']['expression']['type']['the_type'], + reference_algebra.TypeReference.To(args)) + elif '*' in signature: + args = copy(signature['*']) + reference_algebra.UnifyRecordField( + args, fv['field'], + fv['value']['expression']['type']['the_type']) + if isinstance(args.Target(), reference_algebra.BadType): + error_message = ( + ContextualizedError.BuildNiceMessage( + fv['value']['expression']['expression_heritage'].Display(), + 'Predicate %s does not have argument %s, but it was addressed.' % + (color.Format('{warning}%s{end}') % predicate_name, + color.Format('{warning}%s{end}') % fv['field']) + ) + ) + error = reference_algebra.BadType( + ('VERBATIM:' + error_message, + fv['value']['expression']['type']['the_type'].target)) + fv['value']['expression']['type']['the_type'].target = ( + reference_algebra.TypeReference.To(error)) + else: + error_message = ( + ContextualizedError.BuildNiceMessage( + fv['value']['expression']['expression_heritage'].Display(), + 'Predicate %s does not have argument %s, but it was addressed.' % + (color.Format('{warning}%s{end}') % predicate_name, + color.Format('{warning}%s{end}') % fv['field']) + ) + ) + error = reference_algebra.BadType( + ('VERBATIM:' + error_message, + fv['value']['expression']['type']['the_type'].target)) + fv['value']['expression']['type']['the_type'].target = ( + reference_algebra.TypeReference.To(error)) for e in ExpressionsIterator(node): if 'call' in e: p = e['call']['predicate_name'] if p in self.types_of_builtins: - InstillTypes(e['call']['record']['field_value'], - self.types_of_builtins[p], - e['type']['the_type']) + InstillTypes(p, e['call']['record']['field_value'], + self.types_of_builtins[p], e) if 'predicate' in node: p = node['predicate']['predicate_name'] if p in self.types_of_builtins: - InstillTypes(node['predicate']['record']['field_value'], + InstillTypes(p, node['predicate']['record']['field_value'], self.types_of_builtins[p], None) if 'head' in node: p = node['head']['predicate_name'] if p in self.types_of_builtins: - InstillTypes(node['head']['record']['field_value'], + InstillTypes(p, node['head']['record']['field_value'], self.types_of_builtins[p], None) From 7392358777f05e9d7d4fb830d5c21cdc3a323c03 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 17:07:56 -0400 Subject: [PATCH 089/141] Type inference: predicate arguments handling. --- type_inference/research/infer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index aef2ed5d..94f02a55 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -296,11 +296,16 @@ def InstillTypes(predicate_name, output_value_type.target = reference_algebra.TypeReference.To(error) for fv in field_value: - if fv['field'] in signature: + field_name = fv['field'] + if (field_name not in signature and + isinstance(field_name, int) and + 'col%d' % field_name in signature): + field_name = 'col%d' % field_name + if field_name in signature: reference_algebra.Unify( fv['value']['expression']['type']['the_type'], - copy(signature[fv['field']])) - elif fv['field'] == '*': + copy(signature[field_name])) + elif field_name == '*': args = copy(reference_algebra.ClosedRecord(signature)) reference_algebra.Unify( fv['value']['expression']['type']['the_type'], @@ -308,7 +313,7 @@ def InstillTypes(predicate_name, elif '*' in signature: args = copy(signature['*']) reference_algebra.UnifyRecordField( - args, fv['field'], + args, field_name, fv['value']['expression']['type']['the_type']) if isinstance(args.Target(), reference_algebra.BadType): error_message = ( From 527e000bc9bb5b7841189bdc79ee62990e0f73eb Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:28:06 -0400 Subject: [PATCH 090/141] Improving psql execution in Jupyter. --- colab_logica.py | 37 ++++++++++++++++++++++++++----------- compiler/dialects.py | 1 + 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index cb23c85f..feed529c 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -103,7 +103,9 @@ def ConnectToPostgres(mode='interactive'): else: assert False, 'Unknown mode:' + mode connection_json = json.loads(connection_str) - SetDbConnection(psycopg2.connect(**connection_json)) + connection = psycopg2.connect(**connection_json) + connection.autocommit = True + SetDbConnection(connection) def EnsureAuthenticatedUser(): global USER_AUTHENTICATED @@ -158,23 +160,33 @@ def ParseList(line): predicates = [p.strip() for p in line.split(',')] return predicates - +def PostgresExecute(sql, connection): + import psycopg2 + cursor = connection.cursor() + try: + cursor.execute(sql) + except psycopg2.errors.UndefinedTable as e: + raise infer.TypeErrorCaughtException( + infer.ContextualizedError.BuildNiceMessage( + 'Running SQL.', 'Undefined table used: ' + str(e))) + except psycopg2.Error as e: + connection.rollback() + raise e + return cursor + def RunSQL(sql, engine, connection=None, is_final=False): if engine == 'bigquery': client = bigquery.Client(project=PROJECT) return client.query(sql).to_dataframe() elif engine == 'psql': if is_final: - cursor = connection.cursor() - cursor.execute(sql) + cursor = PostgresExecute(sql, connection) rows = cursor.fetchall() df = pandas.DataFrame( rows, columns=[d[0] for d in cursor.description]) return df else: - cursor = connection.cursor() - cursor.execute(sql) - connection.commit() + PostgresExecute(sql, connection) elif engine == 'sqlite': try: if is_final: @@ -301,10 +313,13 @@ def Logica(line, cell, run_query): else: raise Exception('Logica only supports BigQuery, PostgreSQL and SQLite ' 'for now.') - - result_map = concertina_lib.ExecuteLogicaProgram( - executions, sql_runner=sql_runner, sql_engine=engine, - display_mode=DISPLAY_MODE) + try: + result_map = concertina_lib.ExecuteLogicaProgram( + executions, sql_runner=sql_runner, sql_engine=engine, + display_mode=DISPLAY_MODE) + except infer.TypeErrorCaughtException as e: + e.ShowMessage() + return for idx, predicate in enumerate(predicates): t = result_map[predicate] diff --git a/compiler/dialects.py b/compiler/dialects.py index fe36eda8..1d174df8 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -134,6 +134,7 @@ def BuiltInFunctions(self): return { 'Range': '(SELECT ARRAY_AGG(x) FROM GENERATE_SERIES(0, {0} - 1) as x)', 'ToString': 'CAST(%s AS TEXT)', + 'ToInt64': 'CAST(%s AS BIGINT)', 'Element': '({0})[{1} + 1]', 'Size': 'ARRAY_LENGTH(%s, 1)', 'Count': 'COUNT(DISTINCT {0})', From a68c19c593955dc74e9a3ef4a67361c827877c72 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:50:09 -0400 Subject: [PATCH 091/141] Type inference: predicate arguments handling. --- type_inference/research/infer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 94f02a55..5b42a693 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -177,6 +177,14 @@ def UpdateTypes(self, rule): else: value = v['aggregation']['expression'] value_type = value['type']['the_type'] + if field_name not in predicate_signature: + raise TypeErrorCaughtException( + ContextualizedError.BuildNiceMessage( + rule['full_text'], + color.Format( + 'Predicate {warning}%s{end} has ' % predicate_name + + 'inconcistent rules, some include field ') + + color.Format('{warning}%s{end}' % field_name) + ' while others do not.')) reference_algebra.Unify( predicate_signature[field_name], value_type) From e02df936c1359f86a8445d23524c8b678795dc5d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:57:31 -0400 Subject: [PATCH 092/141] Type inference: predicate arguments handling. --- type_inference/research/infer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 5b42a693..82086f5f 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -679,8 +679,9 @@ def PsqlType(self, t): def BuildPsqlDefinitions(self): for t in self.psql_struct_type_name: + arg_name = lambda x: x if isinstance(x, str) else 'col%d' % x args = ', '.join( - f + ' ' + self.PsqlType(v) + arg_name(f) + ' ' + self.PsqlType(v) for f, v in sorted(self.type_map[t].items()) ) self.psql_type_definition[t] = f'create type %s as (%s);' % ( From 46a97c57b612e961aceb091f163e911d0b380d6d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 23:25:40 -0400 Subject: [PATCH 093/141] Built-in functions polishing. --- compiler/dialect_libraries/psql_library.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 0e80cfd3..338f0c95 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -39,4 +39,6 @@ RecordAsJson(r) = SqlExpr( "ROW_TO_JSON({r})", {r:}); + +Fingerprint(s) = SqlExpr("('x' || substr(md5({s}), 1, 16))::bit(64)::bigint", {s:}); """ From 7a7bcb37cc4d73dd6218063557fce9667b556818 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Wed, 19 Jul 2023 23:55:01 -0400 Subject: [PATCH 094/141] Built-in functions polishing. --- type_inference/research/reference_algebra.py | 3 +++ type_inference/research/types_of_builtins.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index e192b08d..3232b08a 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -352,6 +352,9 @@ def CopyConcreteType(self, t): if isinstance(t, dict): c = type(t) return c({k: self.CopyConcreteOrReferenceType(v) for k, v in t.items()}) + if isinstance(t, BadType): + return BadType((self.CopyConcreteOrReferenceType(t[0]), + self.CopyConcreteOrReferenceType(t[1]))) assert False, (t, type(t)) def CopyTypeReference(self, t): diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index f2f18355..99583c78 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -60,7 +60,7 @@ def TypesOfBultins(): 'left': 'Num', 'right': 'Num', 'logica_value': 'Num' - }, + }, 'Num': { 0: 'Num', 'logica_value': 'Num' @@ -153,9 +153,20 @@ def TypesOfBultins(): 1: 'Num', 2: 'Num', 'logica_value': 'Str' + }, + 'Fingerprint': { + 0: 'Str', + 'logica_value': 'Num' + }, + 'Abs': { + 0: 'Num', + 'logica_value': 'Num' } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] + types_of_predicate['Sin'] = types_of_predicate['Cos'] = types_of_predicate['Log' + ] = types_of_predicate['Exp'] = types_of_predicate['Abs'] + types_of_predicate['%'] = types_of_predicate['/'] = types_of_predicate['*'] return { p: {k: reference_algebra.TypeReference(v) for k, v in types.items()} From f0ca28a42a3974d4b823e21c19942a4d61d1191e Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:59:46 -0400 Subject: [PATCH 095/141] Improving psql execution in Jupyter. --- colab_logica.py | 2 ++ compiler/dialects.py | 7 +++++++ compiler/universe.py | 5 +++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index feed529c..daa7b9c8 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -103,7 +103,9 @@ def ConnectToPostgres(mode='interactive'): else: assert False, 'Unknown mode:' + mode connection_json = json.loads(connection_str) + print('Starting connection...') connection = psycopg2.connect(**connection_json) + print('Connection established.') connection.autocommit = True SetDbConnection(connection) diff --git a/compiler/dialects.py b/compiler/dialects.py index 1d174df8..f83e509c 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -38,6 +38,10 @@ def Get(engine): class Dialect(object): pass + # Default methods: + def MaybeCascadingDeletionWord(self): + return '' # No CASCADE is needed by default. + class BigQueryDialect(Dialect): """BigQuery SQL dialect.""" @@ -164,6 +168,9 @@ def GroupBySpecBy(self): def DecorateCombineRule(self, rule, var): return DecorateCombineRule(rule, var) + + def MaybeCascadingDeletionWord(self): + return ' CASCADE' # Need to cascade in PSQL. class Trino(Dialect): diff --git a/compiler/universe.py b/compiler/universe.py index f8894464..1bb146e5 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -1019,8 +1019,9 @@ def TranslateTableAttachedToFile(self, table, ground, external_vocabulary): dependency_sql = self.program.UseFlagsAsParameters(dependency_sql) self.execution.workflow_predicates_stack.pop() maybe_drop_table = ( - 'DROP TABLE IF EXISTS %s;\n' % ground.table_name - if ground.overwrite else '') + 'DROP TABLE IF EXISTS %s%s;\n' % (( + ground.table_name if ground.overwrite else '', + self.execution.dialect.MaybeCascadingDeletionWord()))) export_statement = ( maybe_drop_table + 'CREATE TABLE {name} AS {dependency_sql}'.format( From 34ae61afa0b2532fd895f5377b35cb02554996a0 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 24 Jul 2023 20:02:51 -0700 Subject: [PATCH 096/141] Better rendering of records. --- colab_logica.py | 50 +++++++++++++++++++++- compiler/dialect_libraries/psql_library.py | 3 ++ type_inference/research/infer.py | 1 + 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/colab_logica.py b/colab_logica.py index daa7b9c8..642e0e88 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -16,8 +16,10 @@ """Library for using Logica in CoLab.""" +from decimal import Decimal import getpass import json +import re from .common import color from .common import concertina_lib @@ -164,9 +166,15 @@ def ParseList(line): def PostgresExecute(sql, connection): import psycopg2 + import psycopg2.extras cursor = connection.cursor() try: cursor.execute(sql) + # Make connection aware of the used types. + types = re.findall(r'-- Logica type: (\w*)', sql) + for t in types: + if t != 'logicarecord893574736': # Empty record. + psycopg2.extras.register_composite(t, cursor, globally=True) except psycopg2.errors.UndefinedTable as e: raise infer.TypeErrorCaughtException( infer.ContextualizedError.BuildNiceMessage( @@ -175,7 +183,37 @@ def PostgresExecute(sql, connection): connection.rollback() raise e return cursor - + + +def DigestPsqlType(x): + if isinstance(x, tuple): + return PsqlTypeAsDictionary(x) + if isinstance(x, list) and len(x) > 0: + return PsqlTypeAsList(x) + if isinstance(x, Decimal): + if x.as_integer_ratio()[1] == 1: + return int(x) + else: + return float(x) + return x + + +def PsqlTypeAsDictionary(record): + result = {} + for f in record._asdict(): + a = getattr(record, f) + result[f] = DigestPsqlType(a) + return result + + +def PsqlTypeAsList(a): + e = a[0] + if isinstance(e, tuple): + return [PsqlTypeAsDictionary(i) for i in a] + else: + return a + + def RunSQL(sql, engine, connection=None, is_final=False): if engine == 'bigquery': client = bigquery.Client(project=PROJECT) @@ -186,6 +224,7 @@ def RunSQL(sql, engine, connection=None, is_final=False): rows = cursor.fetchall() df = pandas.DataFrame( rows, columns=[d[0] for d in cursor.description]) + df = df.applymap(DigestPsqlType) return df else: PostgresExecute(sql, connection) @@ -207,6 +246,15 @@ def RunSQL(sql, engine, connection=None, is_final=False): 'for now.') +def Ingress(table_name, csv_file_name): + with open(csv_file_name) as csv_data_io: + cursor = DB_CONNECTION.cursor() + cursor.copy_expert( + 'COPY %s FROM STDIN WITH CSV HEADER' % table_name, + csv_data_io) + DB_CONNECTION.commit() + + class SqliteRunner(object): def __init__(self): self.connection = sqlite3_logica.SqliteConnect() diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 338f0c95..1a8551f2 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -41,4 +41,7 @@ "ROW_TO_JSON({r})", {r:}); Fingerprint(s) = SqlExpr("('x' || substr(md5({s}), 1, 16))::bit(64)::bigint", {s:}); + +Num(a) = a; +Str(a) = a; """ diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 82086f5f..c2fb9f7e 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -688,6 +688,7 @@ def BuildPsqlDefinitions(self): self.psql_struct_type_name[t], args) wrap = lambda n, d: ( + f"-- Logica type: {n}\n" + f"if not exists (select 'I(am) :- I(think)' from pg_type where typname = '{n}') then {d} end if;" ) self.definitions = [ From 983e0c9a2654be74c4f1c2864183cb3f10b95cb4 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 24 Jul 2023 21:00:13 -0700 Subject: [PATCH 097/141] Improving terminal execution. --- colab_logica.py | 56 +++--------------------------- common/psql_logica.py | 73 ++++++++++++++++++++++++++++++++++++++++ tools/run_in_terminal.py | 58 ++++++++++++++++++++----------- 3 files changed, 115 insertions(+), 72 deletions(-) create mode 100644 common/psql_logica.py diff --git a/colab_logica.py b/colab_logica.py index 642e0e88..e1666a85 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -23,6 +23,7 @@ from .common import color from .common import concertina_lib +from .common import psql_logica from .compiler import functors from .compiler import rule_translate @@ -164,55 +165,6 @@ def ParseList(line): predicates = [p.strip() for p in line.split(',')] return predicates -def PostgresExecute(sql, connection): - import psycopg2 - import psycopg2.extras - cursor = connection.cursor() - try: - cursor.execute(sql) - # Make connection aware of the used types. - types = re.findall(r'-- Logica type: (\w*)', sql) - for t in types: - if t != 'logicarecord893574736': # Empty record. - psycopg2.extras.register_composite(t, cursor, globally=True) - except psycopg2.errors.UndefinedTable as e: - raise infer.TypeErrorCaughtException( - infer.ContextualizedError.BuildNiceMessage( - 'Running SQL.', 'Undefined table used: ' + str(e))) - except psycopg2.Error as e: - connection.rollback() - raise e - return cursor - - -def DigestPsqlType(x): - if isinstance(x, tuple): - return PsqlTypeAsDictionary(x) - if isinstance(x, list) and len(x) > 0: - return PsqlTypeAsList(x) - if isinstance(x, Decimal): - if x.as_integer_ratio()[1] == 1: - return int(x) - else: - return float(x) - return x - - -def PsqlTypeAsDictionary(record): - result = {} - for f in record._asdict(): - a = getattr(record, f) - result[f] = DigestPsqlType(a) - return result - - -def PsqlTypeAsList(a): - e = a[0] - if isinstance(e, tuple): - return [PsqlTypeAsDictionary(i) for i in a] - else: - return a - def RunSQL(sql, engine, connection=None, is_final=False): if engine == 'bigquery': @@ -220,14 +172,14 @@ def RunSQL(sql, engine, connection=None, is_final=False): return client.query(sql).to_dataframe() elif engine == 'psql': if is_final: - cursor = PostgresExecute(sql, connection) + cursor = psql_logica.PostgresExecute(sql, connection) rows = cursor.fetchall() df = pandas.DataFrame( rows, columns=[d[0] for d in cursor.description]) - df = df.applymap(DigestPsqlType) + df = df.applymap(psql_logica.DigestPsqlType) return df else: - PostgresExecute(sql, connection) + psql_logica.PostgresExecute(sql, connection) elif engine == 'sqlite': try: if is_final: diff --git a/common/psql_logica.py b/common/psql_logica.py new file mode 100644 index 00000000..97f13c8a --- /dev/null +++ b/common/psql_logica.py @@ -0,0 +1,73 @@ +#!/usr/bin/python +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from decimal import Decimal + +if '.' not in __package__: + from type_inference.research import infer +else: + from ..type_inference.research import infer + + +def PostgresExecute(sql, connection): + import psycopg2 + import psycopg2.extras + cursor = connection.cursor() + try: + cursor.execute(sql) + # Make connection aware of the used types. + types = re.findall(r'-- Logica type: (\w*)', sql) + for t in types: + if t != 'logicarecord893574736': # Empty record. + psycopg2.extras.register_composite(t, cursor, globally=True) + except psycopg2.errors.UndefinedTable as e: + raise infer.TypeErrorCaughtException( + infer.ContextualizedError.BuildNiceMessage( + 'Running SQL.', 'Undefined table used: ' + str(e))) + except psycopg2.Error as e: + connection.rollback() + raise e + return cursor + + +def DigestPsqlType(x): + if isinstance(x, tuple): + return PsqlTypeAsDictionary(x) + if isinstance(x, list) and len(x) > 0: + return PsqlTypeAsList(x) + if isinstance(x, Decimal): + if x.as_integer_ratio()[1] == 1: + return int(x) + else: + return float(x) + return x + + +def PsqlTypeAsDictionary(record): + result = {} + for f in record._asdict(): + a = getattr(record, f) + result[f] = DigestPsqlType(a) + return result + + +def PsqlTypeAsList(a): + e = a[0] + if isinstance(e, tuple): + return [PsqlTypeAsDictionary(i) for i in a] + else: + return a diff --git a/tools/run_in_terminal.py b/tools/run_in_terminal.py index 8a20c8b2..100f6d53 100644 --- a/tools/run_in_terminal.py +++ b/tools/run_in_terminal.py @@ -18,18 +18,26 @@ import json import os +import sys if not __package__ or '.' not in __package__: from common import concertina_lib from compiler import universe from parser_py import parse + from common import psql_logica from common import sqlite3_logica + from compiler import functors + from compiler import rule_translate + from type_inference.research import infer else: from ..common import concertina_lib from ..compiler import universe from ..parser_py import parse + from ..common import psql_logica from ..common import sqlite3_logica - + from ..compiler import functors + from ..compiler import rule_translate + from ..type_inference.research import infer class SqlRunner(object): def __init__(self, engine): @@ -79,19 +87,13 @@ def RunSQL(sql, engine, connection=None, is_final=False, # pandas.read_gbq(sql, project_id=bq_project_id) return list(df.columns), [list(r) for _, r in df.iterrows()] elif engine == 'psql': - import pandas if is_final: - cursor = connection.cursor() - cursor.execute(sql) - rows = cursor.fetchall() - df = pandas.DataFrame( - rows, columns=[d[0] for d in cursor.description]) - connection.close() - return list(df.columns), [list(r) for _, r in df.iterrows()] + cursor = psql_logica.PostgresExecute(sql, connection) + rows = [list(map(psql_logica.DigestPsqlType, row)) + for row in cursor.fetchall()] + return [d[0] for d in cursor.description], rows else: - cursor = connection.cursor() - cursor.execute(sql) - connection.commit() + psql_logica.PostgresExecute(sql, connection) elif engine == 'sqlite': try: if is_final: @@ -112,16 +114,32 @@ def RunSQL(sql, engine, connection=None, is_final=False, def Run(filename, predicate_name): - rules = parse.ParseFile(open(filename).read())['rule'] - program = universe.LogicaProgram(rules) - engine = program.annotations.Engine() + try: + rules = parse.ParseFile(open(filename).read())['rule'] + except parse.ParsingException as parsing_exception: + parsing_exception.ShowMessage() + sys.exit(1) + + + try: + program = universe.LogicaProgram(rules) + engine = program.annotations.Engine() - # This is needed to build the program execution. - unused_sql = program.FormattedPredicateSql(predicate_name) + # This is needed to build the program execution. + unused_sql = program.FormattedPredicateSql(predicate_name) - (header, rows) = concertina_lib.ExecuteLogicaProgram( - [program.execution], SqlRunner(engine), engine, - display_mode='terminal')[predicate_name] + (header, rows) = concertina_lib.ExecuteLogicaProgram( + [program.execution], SqlRunner(engine), engine, + display_mode='terminal')[predicate_name] + except rule_translate.RuleCompileException as rule_compilation_exception: + rule_compilation_exception.ShowMessage() + sys.exit(1) + except functors.FunctorError as functor_exception: + functor_exception.ShowMessage() + sys.exit(1) + except infer.TypeErrorCaughtException as type_error_exception: + type_error_exception.ShowMessage() + sys.exit(1) artistic_table = sqlite3_logica.ArtisticTable(header, rows) return artistic_table From 6a59f3ea27f767e35e52a27d25e3e2a916afe0dd Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:11:55 -0700 Subject: [PATCH 098/141] Better naming in type inference. --- compiler/universe.py | 2 +- type_inference/research/infer.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/compiler/universe.py b/compiler/universe.py index 1bb146e5..3a69a1c7 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -555,7 +555,7 @@ def RunTypechecker(self): typing_engine.InferTypes() type_error_checker = infer.TypeErrorChecker(rules) type_error_checker.CheckForError(mode='raise') - self.predicate_signatures = typing_engine.types_of_builtins + self.predicate_signatures = typing_engine.predicate_signature return typing_engine.typing_preamble def RunMakes(self, rules): diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index c2fb9f7e..1e11a6dd 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -150,7 +150,7 @@ def __init__(self, parsed_rules): self.dependencies = BuildDependencies(self.parsed_rules) self.complexities = BuildComplexities(self.dependencies) self.parsed_rules = list(sorted(self.parsed_rules, key=lambda x: self.complexities[x['head']['predicate_name']])) - self.types_of_builtins = types_of_builtins.TypesOfBultins() + self.predicate_signature = types_of_builtins.TypesOfBultins() self.typing_preamble = None def CollectTypes(self): @@ -160,14 +160,14 @@ def CollectTypes(self): def UpdateTypes(self, rule): predicate_name = rule['head']['predicate_name'] - if predicate_name in self.types_of_builtins: - predicate_signature = self.types_of_builtins[predicate_name] + if predicate_name in self.predicate_signature: + predicate_signature = self.predicate_signature[predicate_name] else: predicate_signature = {} for fv in rule['head']['record']['field_value']: field_name = fv['field'] predicate_signature[field_name] = reference_algebra.TypeReference('Any') - self.types_of_builtins[predicate_name] = predicate_signature + self.predicate_signature[predicate_name] = predicate_signature for fv in rule['head']['record']['field_value']: field_name = fv['field'] @@ -194,7 +194,7 @@ def InferTypes(self): for rule in self.parsed_rules: if rule['head']['predicate_name'][0] == '@': continue - t = TypeInferenceForRule(rule, self.types_of_builtins) + t = TypeInferenceForRule(rule, self.predicate_signature) t.PerformInference() self.UpdateTypes(rule) @@ -204,13 +204,12 @@ def InferTypes(self): def ShowPredicateTypes(self): result_lines = [] - for predicate_name, signature in self.types_of_builtins.items(): + for predicate_name, signature in self.predicate_signature.items(): result_lines.append(RenderPredicateSignature(predicate_name, signature)) return '\n'.join(result_lines) def ConcretizeTypes(node): - # print('>> concretizing', node) if isinstance(node, dict): if 'type' in node: node['type']['the_type'] = reference_algebra.VeryConcreteType( From c7121d83380bfc1f35abf2d8a7850664096e7835 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 11:09:57 -0700 Subject: [PATCH 099/141] Refining. --- compiler/dialects.py | 7 +++++++ compiler/expr_translate.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index f83e509c..73aaa0ea 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -41,6 +41,9 @@ class Dialect(object): # Default methods: def MaybeCascadingDeletionWord(self): return '' # No CASCADE is needed by default. + + def PredicateLiteral(self, predicate_name): + return "'predicate_name:%s'" % predicate_name class BigQueryDialect(Dialect): @@ -75,6 +78,10 @@ def GroupBySpecBy(self): def DecorateCombineRule(self, rule, var): return rule + def PredicateLiteral(self, predicate_name): + return 'STRUCT("%s" AS predicate_name)' % predicate_name + + class SqLiteDialect(Dialect): """SqLite SQL dialect.""" diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 9c457155..2c08525f 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -272,7 +272,7 @@ def NullLiteral(self, literal): def PredicateLiteral(self, literal): if self.convert_to_json: return '{"predicate_name": "%s"}' % (literal['predicate_name']) - return 'STRUCT("%s" AS predicate_name)' % literal['predicate_name'] + return self.dialect.PredicateLiteral(literal['predicate_name']) def Variable(self, variable): if variable['var_name'] in self.vocabulary: From 604890c862ae343bc4c662fe425452ffbe59fa4c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 21:06:33 -0700 Subject: [PATCH 100/141] Refining. --- colab_logica.py | 8 +++++++- compiler/universe.py | 22 +++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index e1666a85..3d4fb5c5 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -80,6 +80,8 @@ DISPLAY_MODE = 'colab' # or colab-text +DEFAULT_ENGINE = 'bigquery' + def SetPreamble(preamble): global PREAMBLE @@ -111,6 +113,8 @@ def ConnectToPostgres(mode='interactive'): print('Connection established.') connection.autocommit = True SetDbConnection(connection) + global DEFAULT_ENGINE + DEFAULT_ENGINE = 'psql' def EnsureAuthenticatedUser(): global USER_AUTHENTICATED @@ -249,7 +253,9 @@ def Logica(line, cell, run_query): e.ShowMessage() return try: - program = universe.LogicaProgram(parsed_rules) + program = universe.LogicaProgram( + parsed_rules, + user_flags={'logica_default_engine': DEFAULT_ENGINE}) except functors.FunctorError as e: e.ShowMessage() return diff --git a/compiler/universe.py b/compiler/universe.py index 3a69a1c7..9a5606db 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -137,6 +137,11 @@ class Annotations(object): def __init__(self, rules, user_flags): # Extracting DefineFlags first, so that we can use flags in @Ground # annotations. + if 'logica_default_engine' in user_flags: + self.default_engine = user_flags['logica_default_engine'] + else: + self.default_engine = 'bigquery' + self.annotations = self.ExtractAnnotations( rules, restrict_to=['@DefineFlag', '@ResetFlagValue']) self.user_flags = user_flags @@ -167,12 +172,14 @@ def BuildFlagValues(self): programmatic_flag_values = {} for flag, a in self.annotations['@ResetFlagValue'].items(): programmatic_flag_values[flag] = a.get('1', '${%s}' % flag) + system_flags = set(['logica_default_engine']) + allowed_flags_set = set(default_values) | system_flags - if not set(self.user_flags) <= set(default_values): + if not set(self.user_flags) <= allowed_flags_set: raise rule_translate.RuleCompileException( 'Undefined flags used: %s' % list( - set(self.user_flags) - set(default_values)), - str(set(self.user_flags) - set(default_values))) + set(self.user_flags) - allowed_flags_set), + str(set(self.user_flags) - allowed_flags_set)) flag_values = default_values flag_values.update(**programmatic_flag_values) flag_values.update(**self.user_flags) @@ -249,19 +256,20 @@ def Dataset(self): return self.ExtractSingleton('@Dataset', 'logica_test') def Engine(self): - engine = self.ExtractSingleton('@Engine', 'bigquery') + engine = self.ExtractSingleton('@Engine', self.default_engine) if engine not in dialects.DIALECTS: AnnotationError('Unrecognized engine: %s' % engine, self.annotations['@Engine'][engine]) return engine def ShouldTypecheck(self): + engine = self.Engine() + if '@Engine' not in self.annotations: - return False + return engine == 'psql' if len(self.annotations['@Engine'].values()) == 0: - return False + return engine == 'psql' - engine = self.Engine() engine_annotation = list(self.annotations['@Engine'].values())[0] if 'type_checking' not in engine_annotation: if engine == 'psql': From 58dfcab147952394f9fc1376f16c25c978f5df5c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:44:03 -0700 Subject: [PATCH 101/141] Refining. --- colab_logica.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 3d4fb5c5..67aa795d 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -387,7 +387,13 @@ def PostgresJumpStart(): from logica import colab_logica from sqlalchemy import create_engine import pandas - engine = create_engine('postgresql+psycopg2://logica:logica@127.0.0.1', pool_recycle=3600) - connection = engine.connect() + # engine = create_engine('postgresql+psycopg2://logica:logica@127.0.0.1', pool_recycle=3600) + # connection = engine.connect() + import psycopg2 + engine = 'psql' + connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica') + print('Connected.') + global DEFAULT_ENGINE + DEFAULT_ENGINE = 'psql' return engine, connection From f4053e8e20f370ada8e6d989aea7a0de5050ac5b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:46:39 -0700 Subject: [PATCH 102/141] Refining. --- colab_logica.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/colab_logica.py b/colab_logica.py index 67aa795d..4e5c8225 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -395,5 +395,7 @@ def PostgresJumpStart(): print('Connected.') global DEFAULT_ENGINE + global DB_CONNECTION DEFAULT_ENGINE = 'psql' + DB_CONNECTION = connection return engine, connection From 8c4539604b46aac0d7e16866abb5d9b83d944a15 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:54:54 -0700 Subject: [PATCH 103/141] Refining. --- colab_logica.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 4e5c8225..6407bd4f 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -224,13 +224,9 @@ class PostgresRunner(object): def __init__(self): global DB_CONNECTION global DB_ENGINE - if DB_CONNECTION: - self.engine = DB_ENGINE - self.connection = DB_CONNECTION - else: - (self.engine, self.connection) = PostgresJumpStart() - DB_ENGINE = self.engine - DB_CONNECTION = self.connection + if not DB_CONNECTION: + PostgresJumpStart() + self.connection = DB_CONNECTION def __call__(self, sql, engine, is_final): return RunSQL(sql, engine, self.connection, is_final) @@ -346,6 +342,14 @@ def Logica(line, cell, run_query): print(' ') # To activate the tabbar. def PostgresJumpStart(): + print("Assuming this is running on Google CoLab in a temporary") + print("environment.") + print("Would you like to install and run postgres?") + user_choice = input('y/N?') + if user_choice != 'y': + print('User declined.') + print('Bailing out.') + return # Install postgresql server. print("Installing and configuring an empty PostgreSQL database.") result = 0 @@ -384,13 +388,7 @@ def PostgresJumpStart(): return print('Installation succeeded. Connecting...') # Connect to the database. - from logica import colab_logica - from sqlalchemy import create_engine - import pandas - # engine = create_engine('postgresql+psycopg2://logica:logica@127.0.0.1', pool_recycle=3600) - # connection = engine.connect() import psycopg2 - engine = 'psql' connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica') print('Connected.') @@ -398,4 +396,3 @@ def PostgresJumpStart(): global DB_CONNECTION DEFAULT_ENGINE = 'psql' DB_CONNECTION = connection - return engine, connection From 1146b5964399a0757d81f7697ae40a1a012ec0fd Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:56:13 -0700 Subject: [PATCH 104/141] Refining. --- colab_logica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colab_logica.py b/colab_logica.py index 6407bd4f..864f9480 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -345,7 +345,7 @@ def PostgresJumpStart(): print("Assuming this is running on Google CoLab in a temporary") print("environment.") print("Would you like to install and run postgres?") - user_choice = input('y/N?') + user_choice = input('y or N? ') if user_choice != 'y': print('User declined.') print('Bailing out.') From dc4957384c7a3b2933e37bd9f142cfdc41a721db Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 28 Jul 2023 18:20:37 -0700 Subject: [PATCH 105/141] Refining. --- .gitignore | 1 + colab_logica.py | 21 ++++++++++++-------- common/psql_logica.py | 6 +----- type_inference/research/reference_algebra.py | 7 ++++++- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 04f18983..4a8ce968 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.DS_Store *.pyc +.ipynb_checkpoints \ No newline at end of file diff --git a/colab_logica.py b/colab_logica.py index 864f9480..652a14e0 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -78,7 +78,10 @@ PREAMBLE = None -DISPLAY_MODE = 'colab' # or colab-text +if hasattr(concertina_lib, 'graphviz'): + DISPLAY_MODE = 'colab' +else: + DISPLAY_MODE = 'colab-text' DEFAULT_ENGINE = 'bigquery' @@ -98,18 +101,20 @@ def SetDbConnection(connection): def ConnectToPostgres(mode='interactive'): import psycopg2 if mode == 'interactive': - print('Please enter PostgreSQL connection config in JSON format.') - print('Example:') - print('{"host": "myhost", "database": "megadb", ' - '"user": "myuser", "password": "42"}') + print('Please enter PostgreSQL URL, or config in JSON format with fields host, database, user and password.') connection_str = getpass.getpass() elif mode == 'environment': connection_str = os.environ.get('LOGICA_PSQL_CONNECTION') else: assert False, 'Unknown mode:' + mode - connection_json = json.loads(connection_str) - print('Starting connection...') - connection = psycopg2.connect(**connection_json) + if connection_str.startswith('postgres'): + print('Connecting to postgresql url...') + connection = psycopg2.connect(connection_str) + else: + print('Parsing connection JSON.') + connection_json = json.loads(connection_str) + print('Issuing connection request...') + connection = psycopg2.connect(**connection_json) print('Connection established.') connection.autocommit = True SetDbConnection(connection) diff --git a/common/psql_logica.py b/common/psql_logica.py index 97f13c8a..b3a1c575 100644 --- a/common/psql_logica.py +++ b/common/psql_logica.py @@ -66,8 +66,4 @@ def PsqlTypeAsDictionary(record): def PsqlTypeAsList(a): - e = a[0] - if isinstance(e, tuple): - return [PsqlTypeAsDictionary(i) for i in a] - else: - return a + return list(map(DigestPsqlType, a)) diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 3232b08a..8da96a3d 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -368,7 +368,12 @@ def Revive(t): if isinstance(t, str): return TypeReference(t) if isinstance(t, dict): - return TypeReference(OpenRecord({k: Revive(v) for k, v in t.items()})) + def ReviveKey(k): + if k in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']: + return int(k) + return k + return TypeReference(OpenRecord( + {ReviveKey(k): Revive(v) for k, v in t.items()})) if isinstance(t, list): return TypeReference(list(map(Revive, t))) if isinstance(t, BadType): From 6e203c199682e0eb54063c60514d080d13e6fba7 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 29 Jul 2023 14:32:21 -0700 Subject: [PATCH 106/141] Refining. --- colab_logica.py | 19 +------------------ common/psql_logica.py | 25 +++++++++++++++++++++++++ compiler/expr_translate.py | 6 ++++++ compiler/rule_translate.py | 3 ++- tools/run_in_terminal.py | 9 +-------- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 652a14e0..c5221da0 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -99,24 +99,7 @@ def SetDbConnection(connection): DB_CONNECTION = connection def ConnectToPostgres(mode='interactive'): - import psycopg2 - if mode == 'interactive': - print('Please enter PostgreSQL URL, or config in JSON format with fields host, database, user and password.') - connection_str = getpass.getpass() - elif mode == 'environment': - connection_str = os.environ.get('LOGICA_PSQL_CONNECTION') - else: - assert False, 'Unknown mode:' + mode - if connection_str.startswith('postgres'): - print('Connecting to postgresql url...') - connection = psycopg2.connect(connection_str) - else: - print('Parsing connection JSON.') - connection_json = json.loads(connection_str) - print('Issuing connection request...') - connection = psycopg2.connect(**connection_json) - print('Connection established.') - connection.autocommit = True + connection = psql_logica.ConnectToPostgres(mode) SetDbConnection(connection) global DEFAULT_ENGINE DEFAULT_ENGINE = 'psql' diff --git a/common/psql_logica.py b/common/psql_logica.py index b3a1c575..443aeda6 100644 --- a/common/psql_logica.py +++ b/common/psql_logica.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import getpass +import json +import os import re from decimal import Decimal @@ -67,3 +70,25 @@ def PsqlTypeAsDictionary(record): def PsqlTypeAsList(a): return list(map(DigestPsqlType, a)) + + +def ConnectToPostgres(mode): + import psycopg2 + if mode == 'interactive': + print('Please enter PostgreSQL URL, or config in JSON format with fields host, database, user and password.') + connection_str = getpass.getpass() + elif mode == 'environment': + connection_str = os.environ.get('LOGICA_PSQL_CONNECTION') + assert connection_str, ( + 'Please provide PSQL connection parameters ' + 'in LOGICA_PSQL_CONNECTION.') + else: + assert False, 'Unknown mode:' + mode + if connection_str.startswith('postgres'): + connection = psycopg2.connect(connection_str) + else: + connection_json = json.loads(connection_str) + connection = psycopg2.connect(**connection_json) + + connection.autocommit = True + return connection \ No newline at end of file diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 2c08525f..4bc4ed2c 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -429,6 +429,12 @@ def GetValueOfField(field_values, field): 'implication': {'if_then': new_if_thens, 'otherwise': new_otherwise}} return self.ConvertToSql(new_expr) + def ConvertToSqlForGroupBy(self, expression): + if 'literal' in expression and 'the_string' in expression['literal']: + # To calm down PSQL: + return f"({self.ConvertToSql(expression)} || '')" + return self.ConvertToSql(expression) + def ConvertToSql(self, expression): """Converting Logica expression into SQL.""" # print('EXPR:', expression) diff --git a/compiler/rule_translate.py b/compiler/rule_translate.py index a0c36025..1edf68f9 100755 --- a/compiler/rule_translate.py +++ b/compiler/rule_translate.py @@ -531,7 +531,8 @@ def AsSql(self, subquery_encoder=None, flag_values=None): for v in ordered_distinct_vars) elif subquery_encoder.execution.dialect.GroupBySpecBy() == 'expr': r += ', '.join( - ql.ConvertToSql(self.select[k]) for k in ordered_distinct_vars + ql.ConvertToSqlForGroupBy(self.select[k]) + for k in ordered_distinct_vars ) else: assert False, 'Broken dialect %s, group by spec: %s' % ( diff --git a/tools/run_in_terminal.py b/tools/run_in_terminal.py index 100f6d53..5dfba444 100644 --- a/tools/run_in_terminal.py +++ b/tools/run_in_terminal.py @@ -57,14 +57,7 @@ def __init__(self, engine): else: credentials, project = None, None if engine == 'psql': - import psycopg2 - if os.environ.get('LOGICA_PSQL_CONNECTION'): - connection_json = json.loads(os.environ.get('LOGICA_PSQL_CONNECTION')) - else: - assert False, ( - 'Please provide PSQL connection parameters ' - 'in LOGICA_PSQL_CONNECTION') - self.connection = psycopg2.connect(**connection_json) + self.connection = psql_logica.ConnectToPostgres('environment') self.bq_credentials = credentials self.bq_project = project From 131e219406be5862c798bef714e6ac76a33987f8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 6 Aug 2023 10:51:38 -1000 Subject: [PATCH 107/141] Adding Bool type and refining. --- common/concertina_lib.py | 2 +- compiler/expr_translate.py | 7 ++-- compiler/functors.py | 23 +++++++++++-- compiler/universe.py | 13 ++++++++ integration_tests/psql_bool_test.l | 18 +++++++++++ logica.py | 14 +++++--- type_inference/research/infer.py | 5 +++ type_inference/research/reference_algebra.py | 10 +++--- type_inference/research/types_of_builtins.py | 34 ++++++++++++++++++-- 9 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 integration_tests/psql_bool_test.l diff --git a/common/concertina_lib.py b/common/concertina_lib.py index 54f5d1fc..dc1b8166 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -38,7 +38,7 @@ def Run(self, action): is_final=(predicate in self.final_predicates)) end = datetime.datetime.now() if self.print_running_predicate: - print(' (%d seconds)' % (end - start).seconds) + print(' (%d ms)' % ((end - start).microseconds / 1000)) if predicate in self.final_predicates: self.final_result[predicate] = result diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 4bc4ed2c..04f59389 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -570,10 +570,13 @@ def ConvertToSql(self, expression): record = expression['record'] record_type = expression.get('type', {}).get('type_name', None) if self.dialect.Name() == 'PostgreSQL' and record_type is None: + rendered_type = expression.get('type', {}).get('rendered_type', None) raise self.exception_maker(color.Format( 'Record needs type in PostgreSQL: ' - '{warning}{record}{end}.', dict( - record=expression['expression_heritage']))) + '{warning}{record}{end} was inferred only ' + 'an incomplete type {warning}{type}{end}.', dict( + record=expression['expression_heritage'], + type=rendered_type))) return self.Record( record, diff --git a/compiler/functors.py b/compiler/functors.py index 7e5ded87..e04a4c16 100755 --- a/compiler/functors.py +++ b/compiler/functors.py @@ -42,6 +42,16 @@ from ..compiler.dialect_libraries import recursion_library from ..parser_py import parse +import datetime + +class Timer: + def __init__(self, name): + self.name = name + self.time = datetime.datetime.now() + + def Stop(self): + print('%s / micros elapsed:' % self.name, (datetime.datetime.now() - self.time).microseconds) + class FunctorError(Exception): """Exception thrown when Make is bad.""" @@ -172,9 +182,18 @@ def BuildArgs(self, functor): while queue: e = queue.popleft() result.add(e) - for a in self.ArgsOf(e): + args_of_e = self.ArgsOf(e) + if isinstance(args_of_e, set): + arg_type = 'final' + else: + arg_type = 'preliminary' + + for a in args_of_e: if a not in result: - queue.append(a) + if arg_type == 'preliminary': + queue.append(a) + else: + result.add(a) del self.args_of[functor] diff --git a/compiler/universe.py b/compiler/universe.py index 9a5606db..2c2105bf 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -418,6 +418,17 @@ def __contains__(self, key): ql.convert_to_json = True annotation = rule['head']['predicate_name'] + # Checking for aggregations. + aggregated_fields = [ + fv['field'] + for fv in rule['head']['record']['field_value'] + if 'aggregation' in fv['value']] + if aggregated_fields: + raise rule_translate.RuleCompileException( + 'Annotation may not use aggregation, but field ' + '%s is aggregated.' % ( + color.Warn(aggregated_fields[0])), + rule_text) field_values_json_str = ql.ConvertToSql( {'record': rule['head']['record']}) try: @@ -518,6 +529,7 @@ def __init__(self, rules, table_aliases=None, user_flags=None): # Infering types if requested. self.typing_preamble = '' self.predicate_signatures = {} + self.typing_engine = None if self.annotations.ShouldTypecheck(): self.typing_preamble = self.RunTypechecker() @@ -561,6 +573,7 @@ def RunTypechecker(self): rules = [r for _, r in self.rules] typing_engine = infer.TypesInferenceEngine(rules) typing_engine.InferTypes() + self.typing_engine = typing_engine type_error_checker = infer.TypeErrorChecker(rules) type_error_checker.CheckForError(mode='raise') self.predicate_signatures = typing_engine.predicate_signature diff --git a/integration_tests/psql_bool_test.l b/integration_tests/psql_bool_test.l new file mode 100644 index 00000000..9d4791e6 --- /dev/null +++ b/integration_tests/psql_bool_test.l @@ -0,0 +1,18 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +Test({a:, b:, c:}) :- a = (1 > 2), b = true, c = (a && b); \ No newline at end of file diff --git a/logica.py b/logica.py index 604baaad..0acb6fc1 100755 --- a/logica.py +++ b/logica.py @@ -188,11 +188,15 @@ def main(argv): return 0 if command == 'show_signatures': - typing_engine = infer.TypesInferenceEngine(parsed_rules) - typing_engine.InferTypes() - print(typing_engine.ShowPredicateTypes()) - type_error_checker = infer.TypeErrorChecker(parsed_rules) - type_error_checker.CheckForError() + try: + logic_program = universe.LogicaProgram(parsed_rules) + if not logic_program.typing_engine: + logic_program.RunTypechecker() + except infer.TypeErrorCaughtException as type_error_exception: + print(logic_program.typing_engine.ShowPredicateTypes()) + type_error_exception.ShowMessage() + return 1 + print(logic_program.typing_engine.ShowPredicateTypes()) return 0 user_flags = ReadUserFlags(parsed_rules, argv[4:]) diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 1e11a6dd..60177c32 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -125,6 +125,8 @@ def ActMindingPodLiterals(node): reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Num')) if 'the_string' in e['literal']: reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Str')) + if 'the_bool' in e['literal']: + reference_algebra.Unify(e['type']['the_type'], reference_algebra.TypeReference('Bool')) def ActClearingTypes(node): if 'type' in node: @@ -648,6 +650,7 @@ def ActPopulatingTypeMap(self, node): t = node['type']['the_type'] t_rendering = reference_algebra.RenderType(t) self.type_map[t_rendering] = t + node['type']['rendered_type'] = t_rendering if isinstance(t, dict) and reference_algebra.IsFullyDefined(t): node['type']['type_name'] = RecordTypeName(t_rendering) if isinstance(t, list) and reference_algebra.IsFullyDefined(t): @@ -669,6 +672,8 @@ def PsqlType(self, t): return 'text' if t == 'Num': return 'numeric' + if t == 'Bool': + return 'bool' if isinstance(t, dict): return RecordTypeName(reference_algebra.RenderType(t)) if isinstance(t, list): diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 8da96a3d..0891738f 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -196,12 +196,14 @@ def Rank(x): return 2 if x == 'Str': return 3 - if isinstance(x, list): + if x == 'Bool': return 4 - if isinstance(x, OpenRecord): + if isinstance(x, list): return 5 - if isinstance(x, ClosedRecord): + if isinstance(x, OpenRecord): return 6 + if isinstance(x, ClosedRecord): + return 7 assert False, 'Bad type: %s' % x @@ -248,7 +250,7 @@ def Unify(a, b): a.target = b return - if concrete_a in ('Num', 'Str'): + if concrete_a in ('Num', 'Str', 'Bool'): if concrete_a == concrete_b: return # It's all fine. # Type error: a is incompatible with b. diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 99583c78..cedfcdde 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -34,7 +34,7 @@ def TypesOfBultins(): '==': { 'left': x, 'right': x, - 'logica_value': 'Any' # TODO: Add Boolean. + 'logica_value': 'Bool' }, '=': { 'left': x, @@ -141,7 +141,7 @@ def TypesOfBultins(): '>': { 'left': x, 'right': x, - 'logica_value': 'Any' + 'logica_value': 'Bool' }, 'ArrayConcat': { 0: [e], @@ -161,12 +161,42 @@ def TypesOfBultins(): 'Abs': { 0: 'Num', 'logica_value': 'Num' + }, + '!': { + 0: 'Bool', + 'logica_value': 'Bool' + }, + '||': { + 'left': 'Bool', + 'right': 'Bool', + 'logica_value': 'Bool' + }, + 'IsNull': { + 0: 'Any', + 'logica_value': 'Bool' + }, + 'ToString': { + 0: 'Any', + 'logica_value': 'Str' + }, + 'ToInt64': { + 0: 'Any', + 'logica_value': 'Num' + }, + 'ToFloat64': { + 0: 'Any', + 'logica_value': 'Num' + }, + 'AnyValue': { + 0: x, + 'logica_value': x } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] types_of_predicate['Sin'] = types_of_predicate['Cos'] = types_of_predicate['Log' ] = types_of_predicate['Exp'] = types_of_predicate['Abs'] types_of_predicate['%'] = types_of_predicate['/'] = types_of_predicate['*'] + types_of_predicate['&&'] = types_of_predicate['||'] return { p: {k: reference_algebra.TypeReference(v) for k, v in types.items()} From a4f84fa03d44fc6648d0d8646ceb642b9921bd50 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 6 Aug 2023 22:56:59 -0700 Subject: [PATCH 108/141] Updating tests incorporating helpful error message hints. --- .../typing_aggregation_test.txt | 6 ++ .../integration_tests/typing_basic_test.txt | 24 +++++ .../typing_combine2_test.txt | 14 +++ .../typing_combines_test.txt | 19 ++++ .../typing_kitchensync_test.txt | 51 +++++++++- .../integration_tests/typing_lists_test.txt | 15 +++ .../integration_tests/typing_nested_test.txt | 17 ++++ .../typing_palindrome_puzzle_test.txt | 98 +++++++++++++++++++ 8 files changed, 243 insertions(+), 1 deletion(-) diff --git a/type_inference/research/integration_tests/typing_aggregation_test.txt b/type_inference/research/integration_tests/typing_aggregation_test.txt index 4cbb51b0..62550ff1 100644 --- a/type_inference/research/integration_tests/typing_aggregation_test.txt +++ b/type_inference/research/integration_tests/typing_aggregation_test.txt @@ -14,6 +14,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -29,6 +30,7 @@ "expression": { "expression_heritage": "t", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -53,6 +55,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -81,6 +84,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -105,6 +109,7 @@ "expression": { "expression_heritage": "t", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -118,6 +123,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 } diff --git a/type_inference/research/integration_tests/typing_basic_test.txt b/type_inference/research/integration_tests/typing_basic_test.txt index 38a94931..ea8af478 100644 --- a/type_inference/research/integration_tests/typing_basic_test.txt +++ b/type_inference/research/integration_tests/typing_basic_test.txt @@ -8,6 +8,7 @@ "left_hand_side": { "expression_heritage": "x", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -31,6 +32,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 14 } @@ -43,6 +45,7 @@ "expression": { "expression_heritage": "z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -57,6 +60,7 @@ }, "expression_heritage": "1 + z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 13 } @@ -74,6 +78,7 @@ "expression": { "expression_heritage": "z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -89,6 +94,7 @@ "expression": { "expression_heritage": "w", "type": { + "rendered_type": "{a: {b: Num}}", "the_type": { "a": { "b": "Num" @@ -109,6 +115,7 @@ "expression": { "expression_heritage": "l", "type": { + "rendered_type": "[Singular]", "the_type": [ "Singular" ], @@ -129,6 +136,7 @@ "element": { "expression_heritage": "e", "type": { + "rendered_type": "Singular", "the_type": "Singular", "type_id": 4 }, @@ -139,6 +147,7 @@ "list": { "expression_heritage": "l", "type": { + "rendered_type": "[Singular]", "the_type": [ "Singular" ], @@ -161,6 +170,7 @@ "record": { "expression_heritage": "w", "type": { + "rendered_type": "{a: {b: Num}}", "the_type": { "a": { "b": "Num" @@ -182,6 +192,7 @@ } }, "type": { + "rendered_type": "{b: Num}", "the_type": { "b": "Num" }, @@ -198,6 +209,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 15 } @@ -210,6 +222,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 16 } @@ -235,6 +248,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -252,6 +266,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 6 } @@ -264,6 +279,7 @@ "expression": { "expression_heritage": "x", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -279,6 +295,7 @@ "expression": { "expression_heritage": "z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -297,6 +314,7 @@ "record": { "expression_heritage": "w", "type": { + "rendered_type": "{a: {b: Num}}", "the_type": { "a": { "b": "Num" @@ -318,6 +336,7 @@ } }, "type": { + "rendered_type": "{b: Num}", "the_type": { "b": "Num" }, @@ -345,6 +364,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 9 } @@ -362,6 +382,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 10 } @@ -386,6 +407,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 12 } @@ -395,6 +417,7 @@ ] }, "type": { + "rendered_type": "{z: Num}", "the_type": { "z": "Num" }, @@ -407,6 +430,7 @@ ] }, "type": { + "rendered_type": "{r: {z: Num}, t: Num, z: Str}", "the_type": { "r": { "z": "Num" diff --git a/type_inference/research/integration_tests/typing_combine2_test.txt b/type_inference/research/integration_tests/typing_combine2_test.txt index 9523c9ad..886d8f8e 100644 --- a/type_inference/research/integration_tests/typing_combine2_test.txt +++ b/type_inference/research/integration_tests/typing_combine2_test.txt @@ -39,6 +39,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 } @@ -59,6 +60,7 @@ "expression_heritage": "x", "type": { "element_type_name": "logicarecord350574256", + "rendered_type": "[{a: {z: Num}}]", "the_type": [ { "a": { @@ -82,6 +84,7 @@ "element": { "expression_heritage": "a", "type": { + "rendered_type": "{z: Num}", "the_type": { "z": "Num" }, @@ -107,6 +110,7 @@ "expression": { "expression_heritage": "z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -119,6 +123,7 @@ ] }, "type": { + "rendered_type": "{z: Num}", "the_type": { "z": "Num" }, @@ -131,6 +136,7 @@ }, "type": { "element_type_name": "logicarecord574638620", + "rendered_type": "[{z: Num}]", "the_type": [ { "z": "Num" @@ -172,6 +178,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "{z: Num}", "the_type": { "z": "Num" }, @@ -187,6 +194,7 @@ ] }, "type": { + "rendered_type": "{a: {z: Num}}", "the_type": { "a": { "z": "Num" @@ -203,6 +211,7 @@ }, "type": { "element_type_name": "logicarecord350574256", + "rendered_type": "[{a: {z: Num}}]", "the_type": [ { "a": { @@ -222,6 +231,7 @@ }, "type": { "element_type_name": "logicarecord350574256", + "rendered_type": "[{a: {z: Num}}]", "the_type": [ { "a": { @@ -245,6 +255,7 @@ "expression": { "expression_heritage": "z", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -273,6 +284,7 @@ "expression_heritage": "x", "type": { "element_type_name": "logicarecord350574256", + "rendered_type": "[{a: {z: Num}}]", "the_type": [ { "a": { @@ -306,6 +318,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 4 } @@ -315,6 +328,7 @@ ] }, "type": { + "rendered_type": "{o: Num}", "the_type": { "o": "Num" }, diff --git a/type_inference/research/integration_tests/typing_combines_test.txt b/type_inference/research/integration_tests/typing_combines_test.txt index cd25ca89..8bf1fe86 100644 --- a/type_inference/research/integration_tests/typing_combines_test.txt +++ b/type_inference/research/integration_tests/typing_combines_test.txt @@ -8,6 +8,7 @@ "left_hand_side": { "expression_heritage": "x", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -25,6 +26,7 @@ "element": { "expression_heritage": "a", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 }, @@ -45,6 +47,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -57,6 +60,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 8 } @@ -69,6 +73,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 9 } @@ -78,6 +83,7 @@ }, "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -110,6 +116,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 }, @@ -123,6 +130,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -135,6 +143,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 4 } @@ -147,6 +156,7 @@ "expression_heritage": "y", "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -166,6 +176,7 @@ "element": { "expression_heritage": "a", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 3 }, @@ -186,6 +197,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 13 } @@ -198,6 +210,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 14 } @@ -207,6 +220,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -239,6 +253,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 3 }, @@ -253,6 +268,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -268,6 +284,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -290,6 +307,7 @@ "expression": { "expression_heritage": "x", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -306,6 +324,7 @@ "expression_heritage": "y", "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], diff --git a/type_inference/research/integration_tests/typing_kitchensync_test.txt b/type_inference/research/integration_tests/typing_kitchensync_test.txt index 79d44d1d..17c9482d 100644 --- a/type_inference/research/integration_tests/typing_kitchensync_test.txt +++ b/type_inference/research/integration_tests/typing_kitchensync_test.txt @@ -39,6 +39,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 } @@ -68,6 +69,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 } @@ -85,6 +87,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -94,6 +97,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -117,6 +121,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 6 } @@ -134,6 +139,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -143,6 +149,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -156,6 +163,7 @@ }, "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -188,6 +196,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 } @@ -217,6 +226,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 } @@ -234,6 +244,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -243,6 +254,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -266,6 +278,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 6 } @@ -283,6 +296,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -292,6 +306,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -305,6 +320,7 @@ }, "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -337,6 +353,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 } @@ -366,6 +383,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 } @@ -383,6 +401,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -392,6 +411,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -415,6 +435,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 6 } @@ -432,6 +453,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -441,6 +463,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -454,6 +477,7 @@ }, "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -484,6 +508,7 @@ "expression": { "expression_heritage": "name", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -500,6 +525,7 @@ "expression_heritage": "items", "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -529,6 +555,7 @@ "expression": { "expression_heritage": "name", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -545,6 +572,7 @@ "expression_heritage": "items", "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -569,6 +597,7 @@ "expression_heritage": "overview", "type": { "element_type_name": "logicarecord980116590", + "rendered_type": "[{item: Str, quantity: Str}]", "the_type": [ { "item": "Str", @@ -597,6 +626,7 @@ "expression": { "expression_heritage": "quantity", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 }, @@ -625,6 +655,7 @@ "expression": { "expression_heritage": "num", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 }, @@ -645,6 +676,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 13 } @@ -656,7 +688,8 @@ }, "expression_heritage": "num > 9", "type": { - "the_type": "Any", + "rendered_type": "Bool", + "the_type": "Bool", "type_id": 11 } }, @@ -668,6 +701,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 12 } @@ -682,12 +716,14 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 10 } } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 9 } @@ -709,6 +745,7 @@ "expression": { "expression_heritage": "num", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 }, @@ -729,6 +766,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 14 } @@ -751,6 +789,7 @@ "expression": { "expression_heritage": "item", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 3 }, @@ -766,6 +805,7 @@ "expression": { "expression_heritage": "num", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 }, @@ -778,6 +818,7 @@ ] }, "type": { + "rendered_type": "{item: Str, num: Num}", "the_type": { "item": "Str", "num": "Num" @@ -790,6 +831,7 @@ "expression_heritage": "items", "type": { "element_type_name": "logicarecord454966611", + "rendered_type": "[{item: Str, num: Num}]", "the_type": [ { "item": "Str", @@ -835,6 +877,7 @@ "expression": { "expression_heritage": "item", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 3 }, @@ -850,6 +893,7 @@ "expression": { "expression_heritage": "quantity", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 }, @@ -862,6 +906,7 @@ ] }, "type": { + "rendered_type": "{item: Str, quantity: Str}", "the_type": { "item": "Str", "quantity": "Str" @@ -877,6 +922,7 @@ }, "type": { "element_type_name": "logicarecord980116590", + "rendered_type": "[{item: Str, quantity: Str}]", "the_type": [ { "item": "Str", @@ -895,6 +941,7 @@ }, "type": { "element_type_name": "logicarecord980116590", + "rendered_type": "[{item: Str, quantity: Str}]", "the_type": [ { "item": "Str", @@ -920,6 +967,7 @@ "expression": { "expression_heritage": "name", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 }, @@ -936,6 +984,7 @@ "expression_heritage": "overview", "type": { "element_type_name": "logicarecord980116590", + "rendered_type": "[{item: Str, quantity: Str}]", "the_type": [ { "item": "Str", diff --git a/type_inference/research/integration_tests/typing_lists_test.txt b/type_inference/research/integration_tests/typing_lists_test.txt index 3a0cfdd5..45b04f86 100644 --- a/type_inference/research/integration_tests/typing_lists_test.txt +++ b/type_inference/research/integration_tests/typing_lists_test.txt @@ -8,6 +8,7 @@ "element": { "expression_heritage": "a", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -28,6 +29,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 6 } @@ -40,6 +42,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -49,6 +52,7 @@ }, "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -69,6 +73,7 @@ "expression_heritage": "b", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -89,6 +94,7 @@ "element": { "expression_heritage": "a", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -100,6 +106,7 @@ "expression_heritage": "b", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -117,6 +124,7 @@ "expression_heritage": "c", "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -139,6 +147,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 9 } @@ -148,6 +157,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -180,6 +190,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 4 } @@ -189,6 +200,7 @@ }, "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -203,6 +215,7 @@ "expression": { "expression_heritage": "a", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -219,6 +232,7 @@ "expression_heritage": "b", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -237,6 +251,7 @@ "expression_heritage": "c", "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], diff --git a/type_inference/research/integration_tests/typing_nested_test.txt b/type_inference/research/integration_tests/typing_nested_test.txt index 631d58e7..ee98a255 100644 --- a/type_inference/research/integration_tests/typing_nested_test.txt +++ b/type_inference/research/integration_tests/typing_nested_test.txt @@ -15,6 +15,7 @@ "expression_heritage": "l", "type": { "element_type_name": "logicarecord762067541", + "rendered_type": "[{a: {b: {c: [Num]}}}]", "the_type": [ { "a": { @@ -76,6 +77,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 9 } @@ -85,6 +87,7 @@ }, "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -96,6 +99,7 @@ ] }, "type": { + "rendered_type": "{c: [Num]}", "the_type": { "c": [ "Num" @@ -110,6 +114,7 @@ ] }, "type": { + "rendered_type": "{b: {c: [Num]}}", "the_type": { "b": { "c": [ @@ -126,6 +131,7 @@ ] }, "type": { + "rendered_type": "{a: {b: {c: [Num]}}}", "the_type": { "a": { "b": { @@ -144,6 +150,7 @@ }, "type": { "element_type_name": "logicarecord762067541", + "rendered_type": "[{a: {b: {c: [Num]}}}]", "the_type": [ { "a": { @@ -169,6 +176,7 @@ "element": { "expression_heritage": "x", "type": { + "rendered_type": "{a: {b: {c: [Num]}}}", "the_type": { "a": { "b": { @@ -189,6 +197,7 @@ "expression_heritage": "l", "type": { "element_type_name": "logicarecord762067541", + "rendered_type": "[{a: {b: {c: [Num]}}}]", "the_type": [ { "a": { @@ -220,6 +229,7 @@ "expression_heritage": "c", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -246,6 +256,7 @@ "record": { "expression_heritage": "x", "type": { + "rendered_type": "{a: {b: {c: [Num]}}}", "the_type": { "a": { "b": { @@ -271,6 +282,7 @@ } }, "type": { + "rendered_type": "{b: {c: [Num]}}", "the_type": { "b": { "c": [ @@ -291,6 +303,7 @@ } }, "type": { + "rendered_type": "{c: [Num]}", "the_type": { "c": [ "Num" @@ -310,6 +323,7 @@ }, "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -327,6 +341,7 @@ "element": { "expression_heritage": "y", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -338,6 +353,7 @@ "expression_heritage": "c", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -363,6 +379,7 @@ "expression": { "expression_heritage": "y", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, diff --git a/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt index c349d584..3cc9fac1 100644 --- a/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt +++ b/type_inference/research/integration_tests/typing_palindrome_puzzle_test.txt @@ -39,6 +39,7 @@ } }, "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 0 } @@ -64,6 +65,7 @@ "expression": { "expression_heritage": "word", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 1 }, @@ -82,6 +84,7 @@ "element": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -108,6 +111,7 @@ "expression": { "expression_heritage": "word", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 1 }, @@ -122,6 +126,7 @@ }, "expression_heritage": "Length(word)", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -134,6 +139,7 @@ "expression_heritage": "Range(Length(word))", "type": { "element_type_name": "numeric", + "rendered_type": "[Num]", "the_type": [ "Num" ], @@ -156,6 +162,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -179,6 +186,7 @@ "expression": { "expression_heritage": "word", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 1 }, @@ -202,6 +210,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -222,6 +231,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 4 } @@ -233,6 +243,7 @@ }, "expression_heritage": "i + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 3 } @@ -250,6 +261,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -261,6 +273,7 @@ }, "expression_heritage": "Substr(word, i + 1, 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 2 } @@ -306,6 +319,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -321,6 +335,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -344,6 +359,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -358,6 +374,7 @@ }, "expression_heritage": "Char(i)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 1 } @@ -385,6 +402,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -399,6 +417,7 @@ }, "expression_heritage": "Char(i)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 8 } @@ -422,6 +441,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -442,6 +462,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 11 } @@ -453,6 +474,7 @@ }, "expression_heritage": "i + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 10 } @@ -464,6 +486,7 @@ }, "expression_heritage": "Char(i + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 9 } @@ -484,6 +507,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -507,6 +531,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -527,6 +552,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 } @@ -538,6 +564,7 @@ }, "expression_heritage": "i + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 } @@ -566,6 +593,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -580,6 +608,7 @@ }, "expression_heritage": "Char(i)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 } @@ -608,6 +637,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -628,6 +658,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 7 } @@ -639,6 +670,7 @@ }, "expression_heritage": "i + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 6 } @@ -650,6 +682,7 @@ }, "expression_heritage": "Char(i + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 5 } @@ -661,6 +694,7 @@ }, "expression_heritage": "Char(i) ++ Char(i + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 3 } @@ -686,6 +720,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -701,6 +736,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -735,6 +771,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -755,6 +792,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 18 } @@ -766,6 +804,7 @@ }, "expression_heritage": "i - 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 17 } @@ -777,6 +816,7 @@ }, "expression_heritage": "Char(i - 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 15 } @@ -800,6 +840,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -820,6 +861,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 20 } @@ -831,6 +873,7 @@ }, "expression_heritage": "j + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 19 } @@ -842,6 +885,7 @@ }, "expression_heritage": "Char(j + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 16 } @@ -870,6 +914,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -890,6 +935,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 3 } @@ -901,6 +947,7 @@ }, "expression_heritage": "i - 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 } @@ -921,6 +968,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -941,6 +989,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 5 } @@ -952,6 +1001,7 @@ }, "expression_heritage": "j + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 4 } @@ -996,6 +1046,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1016,6 +1067,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 10 } @@ -1027,6 +1079,7 @@ }, "expression_heritage": "i - 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 9 } @@ -1038,6 +1091,7 @@ }, "expression_heritage": "Char(i - 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 8 } @@ -1058,6 +1112,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1073,6 +1128,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1087,6 +1143,7 @@ }, "expression_heritage": "Palindrome(i, j)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 11 } @@ -1098,6 +1155,7 @@ }, "expression_heritage": "Char(i - 1) ++ Palindrome(i, j)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 7 } @@ -1126,6 +1184,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1146,6 +1205,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 14 } @@ -1157,6 +1217,7 @@ }, "expression_heritage": "j + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 13 } @@ -1168,6 +1229,7 @@ }, "expression_heritage": "Char(j + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 12 } @@ -1179,6 +1241,7 @@ }, "expression_heritage": "Char(i - 1) ++ Palindrome(i, j) ++ Char(j + 1)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 6 } @@ -1224,6 +1287,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1239,6 +1303,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1274,6 +1339,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1289,6 +1355,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1303,6 +1370,7 @@ }, "expression_heritage": "Palindrome(i, j)", "type": { + "rendered_type": "Str", "the_type": "Str", "type_id": 4 } @@ -1312,6 +1380,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -1323,6 +1392,7 @@ ] }, "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1350,6 +1420,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1365,6 +1436,7 @@ "expression": { "expression_heritage": "k", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1406,6 +1478,7 @@ "expression": { "expression_heritage": "i", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1421,6 +1494,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 }, @@ -1435,6 +1509,7 @@ }, "expression_heritage": "Path(i, j)", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1454,6 +1529,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -1487,6 +1563,7 @@ "expression": { "expression_heritage": "j", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 2 }, @@ -1507,6 +1584,7 @@ } }, "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 10 } @@ -1518,6 +1596,7 @@ }, "expression_heritage": "j + 1", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 9 } @@ -1530,6 +1609,7 @@ "expression": { "expression_heritage": "k", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1544,6 +1624,7 @@ }, "expression_heritage": "Path(j + 1, k)", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1563,6 +1644,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -1577,6 +1659,7 @@ "expression_heritage": "ArrayConcat(Path(i, j).path, Path(j + 1, k).path)", "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -1588,6 +1671,7 @@ ] }, "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1612,6 +1696,7 @@ "left_hand_side": { "expression_heritage": "path", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1633,6 +1718,7 @@ }, "expression_heritage": "Path()", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1676,6 +1762,7 @@ "expression": { "expression_heritage": "path", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1707,6 +1794,7 @@ "record": { "expression_heritage": "path", "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1729,6 +1817,7 @@ }, "type": { "element_type_name": "text", + "rendered_type": "[Str]", "the_type": [ "Str" ], @@ -1742,6 +1831,7 @@ }, "expression_heritage": "Size(path.path)", "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 3 } @@ -1753,6 +1843,7 @@ }, "expression_heritage": "path -> Size(path.path)", "type": { + "rendered_type": "{arg: {path: [Str]}, value: Num}", "the_type": { "arg": { "path": [ @@ -1771,6 +1862,7 @@ } }, "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1801,6 +1893,7 @@ "value": { "expression": { "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1815,6 +1908,7 @@ "value": { "expression": { "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1829,6 +1923,7 @@ "value": { "expression": { "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" @@ -1861,6 +1956,7 @@ "value": { "expression": { "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 0 }, @@ -1875,6 +1971,7 @@ "value": { "expression": { "type": { + "rendered_type": "Num", "the_type": "Num", "type_id": 1 }, @@ -1889,6 +1986,7 @@ "value": { "expression": { "type": { + "rendered_type": "{path: [Str]}", "the_type": { "path": [ "Str" From 9f8bab95a1d2805dbe0f1cdac3940dcd2c77b7ac Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sun, 6 Aug 2023 23:44:13 -0700 Subject: [PATCH 109/141] Making table rows to records conversion more robust. --- compiler/dialects.py | 15 +++++--- compiler/expr_translate.py | 47 ++++++++++++++++++++--- compiler/rule_translate.py | 19 ++++++++- compiler/universe.py | 5 +++ integration_tests/run_tests.py | 1 + integration_tests/sqlite_records_test.l | 26 +++++++++++++ integration_tests/sqlite_records_test.txt | 8 ++++ 7 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 integration_tests/sqlite_records_test.l create mode 100644 integration_tests/sqlite_records_test.txt diff --git a/compiler/dialects.py b/compiler/dialects.py index 73aaa0ea..2d189991 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -60,7 +60,7 @@ def InfixOperators(self): '++': 'CONCAT(%s, %s)', } - def Subscript(self, record, subscript): + def Subscript(self, record, subscript, record_is_table): return '%s.%s' % (record, subscript) def LibraryProgram(self): @@ -120,8 +120,11 @@ def InfixOperators(self): 'in': 'IN_LIST(%s, %s)' } - def Subscript(self, record, subscript): - return 'JSON_EXTRACT(%s, "$.%s")' % (record, subscript) + def Subscript(self, record, subscript, record_is_table): + if record_is_table: + return '%s.%s' % (record, subscript) + else: + return 'JSON_EXTRACT(%s, "$.%s")' % (record, subscript) def LibraryProgram(self): return sqlite_library.library @@ -158,7 +161,7 @@ def InfixOperators(self): '++': 'CONCAT(%s, %s)', } - def Subscript(self, record, subscript): + def Subscript(self, record, subscript, record_is_table): return '(%s).%s' % (record, subscript) def LibraryProgram(self): @@ -201,7 +204,7 @@ def InfixOperators(self): '++': 'CONCAT(%s, %s)', } - def Subscript(self, record, subscript): + def Subscript(self, record, subscript, record_is_table): return '%s.%s' % (record, subscript) def LibraryProgram(self): @@ -239,7 +242,7 @@ def InfixOperators(self): '++': 'CONCAT(%s, %s)', } - def Subscript(self, record, subscript): + def Subscript(self, record, subscript, record_is_table): return '%s.%s' % (record, subscript) def LibraryProgram(self): diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 04f59389..84134b0b 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -235,10 +235,10 @@ def Function(self, f, args): def Infix(self, op, args): return op % (args['left'], args['right']) - def Subscript(self, record, subscript): + def Subscript(self, record, subscript, record_is_table): if isinstance(subscript, int): subscript = 'col%d' % subscript - return self.dialect.Subscript(record, subscript) + return self.dialect.Subscript(record, subscript, record_is_table) def IntLiteral(self, literal): return str(literal['number']) @@ -274,7 +274,33 @@ def PredicateLiteral(self, literal): return '{"predicate_name": "%s"}' % (literal['predicate_name']) return self.dialect.PredicateLiteral(literal['predicate_name']) - def Variable(self, variable): + def VariableMaybeTableSQLite(self, variable, expression_type): + def MakeSubscript(subscripted_variable, subscripted_field): + return {'expression': + {'subscript': {'record': {'variable': + {'var_name': subscripted_variable, + 'dont_expand': True}}, + 'subscript': { + 'literal': {'the_symbol': { + 'symbol': subscripted_field}}}}}} + expr = self.vocabulary[variable['var_name']] + if '.' not in expr and 'dont_expand' not in variable: + + if not isinstance(expression_type, dict): + raise self.exception_maker( + 'Could not create record ' + color.Warn(expr) + + '. Type inference is ' + + 'required to convert table rows to records in SQLite.') + return self.ConvertToSql({'record': {'field_value': [ + {'field': k, + 'value': MakeSubscript(variable['var_name'], k)} + for k in sorted(expression_type) + ]}}) + return expr + + def Variable(self, variable, expression_type): + if self.dialect.Name() == 'SqLite': + return self.VariableMaybeTableSQLite(variable, expression_type) if variable['var_name'] in self.vocabulary: return self.vocabulary[variable['var_name']] else: @@ -435,12 +461,23 @@ def ConvertToSqlForGroupBy(self, expression): return f"({self.ConvertToSql(expression)} || '')" return self.ConvertToSql(expression) + def ExpressionIsTable(self, expression): + return ( + 'variable' in expression and + 'dont_expand' in expression['variable'] and + self.VariableIsTable(expression['variable']['var_name'])) + + def VariableIsTable(self, variable_name): + return '.' not in self.vocabulary[variable_name] + def ConvertToSql(self, expression): """Converting Logica expression into SQL.""" # print('EXPR:', expression) # Variables. if 'variable' in expression: - return self.Variable(expression['variable']) + the_type = expression.get('type', {}).get('the_type', 'Any') + return self.Variable(expression['variable'], + the_type) # Literals. if 'literal' in expression: @@ -564,7 +601,7 @@ def ConvertToSql(self, expression): return simplified_sub # Couldn't optimize, just return the '.' expression. record = self.ConvertToSql(sub['record']) - return self.Subscript(record, subscript) + return self.Subscript(record, subscript, self.ExpressionIsTable(sub['record'])) if 'record' in expression: record = expression['record'] diff --git a/compiler/rule_translate.py b/compiler/rule_translate.py index 1edf68f9..4a563b79 100755 --- a/compiler/rule_translate.py +++ b/compiler/rule_translate.py @@ -206,6 +206,13 @@ def __init__(self, names_allocator=None, external_vocabulary=None, self.full_rule_text = None self.distinct_denoted = None + def SelectAsRecord(self): + return {'record': { + 'field_value': [{ + 'field': k, + 'value': {'expression': v} + } for k, v in sorted(self.select.items())]}} + def OwnVarsVocabulary(self): """Returns a map: logica variable -> SQL expression with the value.""" def TableAndFieldToSql(table, field): @@ -476,7 +483,11 @@ def AsSql(self, subquery_encoder=None, flag_values=None): for k, v in self.select.items(): if k == '*': - fields.append('%s.*' % ql.ConvertToSql(v)) + if 'variable' in v: + v['variable']['dont_expand'] = True # For SQLite. + fields.append( + subquery_encoder.execution.dialect.Subscript( + ql.ConvertToSql(v), '*', True)) else: fields.append('%s AS %s' % (ql.ConvertToSql(v), LogicaFieldToSqlField(k))) r += ',\n'.join(' ' + f for f in fields) @@ -504,6 +515,9 @@ def AsSql(self, subquery_encoder=None, flag_values=None): tables.append(sql) self.SortUnnestings() for element, the_list in self.unnestings: + if 'variable' in element: + # To prevent SQLite record unfolding. + element['variable']['dont_expand'] = True tables.append( subquery_encoder.execution.dialect.UnnestPhrase().format( ql.ConvertToSql(the_list), ql.ConvertToSql(element))) @@ -610,7 +624,8 @@ def ExtractInclusionStructure(inclusion, s): 'value': { 'expression': { 'variable': { - 'var_name': var_name + 'var_name': var_name, + 'dont_expand': True } } } diff --git a/compiler/universe.py b/compiler/universe.py index 2c2105bf..1351f5bf 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -926,6 +926,11 @@ def RunInjections(self, s, allocator): } } }) + elif table_var == '*': + s.vars_unification.append({ + 'left': {'variable': {'var_name': clause_var}}, + 'right': rs.SelectAsRecord() + }) else: extra_hint = '' if table_var != '*' else ( ' Are you using .. for injectible predicate? ' diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index f8ec2220..e40f4221 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -69,6 +69,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("rec_small_cycle_test") RunTest("rec_cycle_test") + RunTest("sqlite_records_test") RunTest("sqlite_is_test") RunTest("sqlite_record_assembler") RunTest("sqlite_assignment_test") diff --git a/integration_tests/sqlite_records_test.l b/integration_tests/sqlite_records_test.l new file mode 100644 index 00000000..c71e89e4 --- /dev/null +++ b/integration_tests/sqlite_records_test.l @@ -0,0 +1,26 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("sqlite", type_checking: true); + +T(a: 1, b: 2); +T(a: 3, b: 4); + +D(..r) :- T(..r), r.a == 1; + +E(two_a: 2 * a) :- T(a:); + +@OrderBy(Test, "a", "ta"); +Test(a: r.a, b: r.b, r:, c:, two_a_record:, ta: two_a_record.two_a) :- D(a: c), T(..r), E(..two_a_record); \ No newline at end of file diff --git a/integration_tests/sqlite_records_test.txt b/integration_tests/sqlite_records_test.txt new file mode 100644 index 00000000..944f5419 --- /dev/null +++ b/integration_tests/sqlite_records_test.txt @@ -0,0 +1,8 @@ ++---+---+---------------+---+--------------+----+ +| a | b | r | c | two_a_record | ta | ++---+---+---------------+---+--------------+----+ +| 1 | 2 | {"a":1,"b":2} | 1 | {"two_a":2} | 2 | +| 1 | 2 | {"a":1,"b":2} | 1 | {"two_a":6} | 6 | +| 3 | 4 | {"a":3,"b":4} | 1 | {"two_a":2} | 2 | +| 3 | 4 | {"a":3,"b":4} | 1 | {"two_a":6} | 6 | ++---+---+---------------+---+--------------+----+ \ No newline at end of file From 7164c91b96f8d29264d9e6aa049b0937610b4673 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 7 Aug 2023 18:52:02 -0700 Subject: [PATCH 110/141] Trying to elliminate race condition at initialization. --- compiler/expr_translate.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 84134b0b..92265421 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -96,8 +96,8 @@ class QL(object): '&&': '%s AND %s', '%': 'MOD(%s, %s)' } - BULK_FUNCTIONS = {} - BULK_FUNCTIONS_ARITY_RANGE = {} + BULK_FUNCTIONS = None + BULK_FUNCTIONS_ARITY_RANGE = None # When adding any analytic functions please check that ConvertAnalytic # function handles them correctly. @@ -188,18 +188,22 @@ def CamelCase(s): reader = processed_functions.GetCsv() header = next(reader) + bulk_functions = {} + bulk_functions_arity_range = {} for row in reader: row = dict(list(zip(header, row))) if row['function'][0] == '$': # TODO: Process operators. continue function_name = CamelCase(row['function']) - cls.BULK_FUNCTIONS[function_name] = ( + bulk_functions[function_name] = ( '%s(%s)' % (row['sql_function'], '%s')) - cls.BULK_FUNCTIONS_ARITY_RANGE[function_name] = ( + bulk_functions_arity_range[function_name] = ( int(row['min_args']), float('inf') if row['has_repeated_args'] == '1' else int(row['max_args'])) + cls.BULK_FUNCTIONS = bulk_functions + cls.BULK_FUNCTIONS_ARITY_RANGE = bulk_functions_arity_range def BuiltInFunctionArityRange(self, f): """Returns arity of the built-in function.""" From 83e515a088a628948e710ee8f6273c3177772252 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 8 Aug 2023 00:25:46 -0700 Subject: [PATCH 111/141] Patching ShortestPath bug. --- compiler/rule_translate.py | 7 +++- compiler/universe.py | 2 +- integration_tests/run_tests.py | 1 + integration_tests/sqlite_shortest_path_test.l | 38 +++++++++++++++++++ .../sqlite_shortest_path_test.txt | 24 ++++++++++++ 5 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 integration_tests/sqlite_shortest_path_test.l create mode 100644 integration_tests/sqlite_shortest_path_test.txt diff --git a/compiler/rule_translate.py b/compiler/rule_translate.py index 4a563b79..cdd95541 100755 --- a/compiler/rule_translate.py +++ b/compiler/rule_translate.py @@ -291,7 +291,8 @@ def ReplaceVariableEverywhere(self, u_left, u_right): ReplaceVariable(u_left, u_right, self.vars_unification) ReplaceVariable(u_left, u_right, self.constraints) - def ElliminateInternalVariables(self, assert_full_ellimination=False): + # TODO: Parameter unfold_recods just patches some bug. Careful review is needed. + def ElliminateInternalVariables(self, assert_full_ellimination=False, unfold_records=True): """Elliminates internal variables via substitution.""" variables = self.InternalVariables() while True: @@ -316,7 +317,9 @@ def ElliminateInternalVariables(self, assert_full_ellimination=False): self.ReplaceVariableEverywhere(u_left, u_right) done = False # Assignments to variables in record fields. - if True: # Confirm that unwraping works and make this unconditional. + if unfold_records: # Confirm that unwraping works and make this unconditional. + # Unwrapping goes wild sometimes. Letting it go right to left only. + # for k, r in [['left', 'right']]: for k, r in [['left', 'right'], ['right', 'left']]: if u[k] == u[r]: continue diff --git a/compiler/universe.py b/compiler/universe.py index 1351f5bf..e51acd2d 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -899,7 +899,7 @@ def RunInjections(self, s, allocator): [r] = rules rs = rule_translate.ExtractRuleStructure( r, allocator, None) - rs.ElliminateInternalVariables(assert_full_ellimination=False) + rs.ElliminateInternalVariables(assert_full_ellimination=False, unfold_records=False) new_tables.update(rs.tables) InjectStructure(s, rs) diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index e40f4221..4b4009b6 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -69,6 +69,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("rec_small_cycle_test") RunTest("rec_cycle_test") + RunTest("sqlite_shortest_path_test") RunTest("sqlite_records_test") RunTest("sqlite_is_test") RunTest("sqlite_record_assembler") diff --git a/integration_tests/sqlite_shortest_path_test.l b/integration_tests/sqlite_shortest_path_test.l new file mode 100644 index 00000000..fc71bd44 --- /dev/null +++ b/integration_tests/sqlite_shortest_path_test.l @@ -0,0 +1,38 @@ +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This example was causing record unfolding go wild. + +@Engine("sqlite"); + +Edge("a", "b"); +Edge("b", "c"); +Edge("c", "d"); +Edge("d", "e"); +Edge("d", "a"); + +ShortestPath(open_path) = ArgMin(k -> v).nodes :- + k == {nodes: open_path}, + v == Size(open_path); + +OpenPath(start, end) ShortestPath= [start] :- + (Edge(start, end) | Edge(end, start)); + +OpenPath(start, end) ShortestPath= ArrayConcat(path_one, path_two) :- + OpenPath(start, intermediate_node) = path_one, + OpenPath(intermediate_node, end) = path_two, + start != end; + +Test(a, b, OpenPath(a, b)); \ No newline at end of file diff --git a/integration_tests/sqlite_shortest_path_test.txt b/integration_tests/sqlite_shortest_path_test.txt new file mode 100644 index 00000000..a50de50d --- /dev/null +++ b/integration_tests/sqlite_shortest_path_test.txt @@ -0,0 +1,24 @@ ++------+------+-----------------+ +| col0 | col1 | col2 | ++------+------+-----------------+ +| a | b | ["a"] | +| a | c | ["a", "b"] | +| a | d | ["a"] | +| a | e | ["a", "d"] | +| b | a | ["b"] | +| b | c | ["b"] | +| b | d | ["b", "a"] | +| b | e | ["b", "a", "d"] | +| c | a | ["c", "b"] | +| c | b | ["c"] | +| c | d | ["c"] | +| c | e | ["c", "d"] | +| d | a | ["d"] | +| d | b | ["d", "a"] | +| d | c | ["d"] | +| d | e | ["d"] | +| e | a | ["e", "d"] | +| e | b | ["e", "d", "a"] | +| e | c | ["e", "d"] | +| e | d | ["e"] | ++------+------+-----------------+ \ No newline at end of file From 3427c175b4981bfd8fef5bd33a535b3ecb6e1f3b Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 8 Aug 2023 17:40:14 -0700 Subject: [PATCH 112/141] Allow lists are args of ArgMax. --- compiler/dialect_libraries/psql_library.py | 8 ++++---- compiler/universe.py | 8 ++++++++ integration_tests/psql_argmin_list_test.l | 21 ++++++++++++++++++++ type_inference/research/infer.py | 20 ++++++++++++------- type_inference/research/reference_algebra.py | 4 ++-- type_inference/research/types_of_builtins.py | 8 ++++---- 6 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 integration_tests/psql_argmin_list_test.l diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 1a8551f2..312f3842 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -18,12 +18,12 @@ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; -ArgMin(a) = SqlExpr("(ARRAY_AGG({arg} order by {value}))[1]", - {arg: a.arg, value: a.value}); +ArgMin(a) = (SqlExpr("(ARRAY_AGG({arg} order by {value}))[1]", + {arg: {argpod: a.arg}, value: a.value})).argpod; -ArgMax(a) = SqlExpr( +ArgMax(a) = (SqlExpr( "(ARRAY_AGG({arg} order by {value} desc))[1]", - {arg: a.arg, value: a.value}); + {arg: {argpod: a.arg}, value: a.value})).argpod; ArgMaxK(a, l) = SqlExpr( "(ARRAY_AGG({arg} order by {value} desc))[1:{lim}]", diff --git a/compiler/universe.py b/compiler/universe.py index e51acd2d..487cf229 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -528,6 +528,7 @@ def __init__(self, rules, table_aliases=None, user_flags=None): # Infering types if requested. self.typing_preamble = '' + self.required_type_definitions = {} self.predicate_signatures = {} self.typing_engine = None if self.annotations.ShouldTypecheck(): @@ -577,6 +578,7 @@ def RunTypechecker(self): type_error_checker = infer.TypeErrorChecker(rules) type_error_checker.CheckForError(mode='raise') self.predicate_signatures = typing_engine.predicate_signature + self.required_type_definitions.update(typing_engine.collector.definitions) return typing_engine.typing_preamble def RunMakes(self, rules): @@ -798,6 +800,8 @@ def InitializeExecution(self, main_predicate): []) self.execution.dependencies_of = self.functors.args_of self.execution.dialect = dialects.Get(self.annotations.Engine()) + + def UpdateExecutionWithTyping(self): if self.execution.dialect.Name() == 'PostgreSQL': self.execution.preamble += '\n' + self.typing_preamble @@ -816,6 +820,8 @@ def FormattedPredicateSql(self, name, allocator=None): else: sql = self.PredicateSql(name, allocator) + self.UpdateExecutionWithTyping() + assert self.execution.workflow_predicates_stack == [name], ( 'Logica internal error: unexpected workflow stack: %s' % self.execution.workflow_predicates_stack) @@ -977,6 +983,8 @@ def SingleRuleSql(self, rule, s.UnificationsToConstraints() type_inference = infer.TypeInferenceForStructure(s, self.predicate_signatures) type_inference.PerformInference() + self.required_type_definitions.update(type_inference.collector.definitions) + self.typing_preamble = infer.BuildPreamble(self.required_type_definitions) try: sql = s.AsSql(self.MakeSubqueryTranslator(allocator), self.flag_values) diff --git a/integration_tests/psql_argmin_list_test.l b/integration_tests/psql_argmin_list_test.l new file mode 100644 index 00000000..eecd757f --- /dev/null +++ b/integration_tests/psql_argmin_list_test.l @@ -0,0 +1,21 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +T1() ArgMin= [i] -> (i - 3) ^ 2 :- i in Range(10); +T2() ArgMax= [i, i + 1] -> (i - 3) ^ 2 :- i in Range(10); + +Test(T1(), T2()); \ No newline at end of file diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 60177c32..571d755c 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -154,11 +154,13 @@ def __init__(self, parsed_rules): self.parsed_rules = list(sorted(self.parsed_rules, key=lambda x: self.complexities[x['head']['predicate_name']])) self.predicate_signature = types_of_builtins.TypesOfBultins() self.typing_preamble = None + self.collector = None def CollectTypes(self): collector = TypeCollector(self.parsed_rules) collector.CollectTypes() self.typing_preamble = collector.typing_preamble + self.collector = collector def UpdateTypes(self, rule): predicate_name = rule['head']['predicate_name'] @@ -477,6 +479,7 @@ class TypeInferenceForStructure: def __init__(self, structure, signatures): self.structure = structure self.signatures = signatures + self.collector = None def PerformInference(self): quazy_rule = self.BuildQuazyRule() @@ -485,10 +488,11 @@ def PerformInference(self): inferencer = TypeInferenceForRule(quazy_rule, self.signatures) inferencer.PerformInference() Walk(quazy_rule, ActRecallingTypes) - + Walk(quazy_rule, ConcretizeTypes) collector = TypeCollector([quazy_rule]) collector.CollectTypes() + self.collector = collector import json # @@ -644,7 +648,7 @@ def __init__(self, parsed_rules): self.psql_type_definition = {} self.definitions = [] self.typing_preamble = '' - + def ActPopulatingTypeMap(self, node): if 'type' in node: t = node['type']['the_type'] @@ -656,7 +660,7 @@ def ActPopulatingTypeMap(self, node): if isinstance(t, list) and reference_algebra.IsFullyDefined(t): [e] = t node['type']['element_type_name'] = self.PsqlType(e) - + def CollectTypes(self): Walk(self.parsed_rules, self.ActPopulatingTypeMap) for t in self.type_map: @@ -695,8 +699,10 @@ def BuildPsqlDefinitions(self): f"-- Logica type: {n}\n" + f"if not exists (select 'I(am) :- I(think)' from pg_type where typname = '{n}') then {d} end if;" ) - self.definitions = [ - wrap(self.psql_struct_type_name[t], self.psql_type_definition[t]) - for t in sorted(self.psql_struct_type_name, key=len)] + self.definitions = { + t: wrap(self.psql_struct_type_name[t], self.psql_type_definition[t]) + for t in sorted(self.psql_struct_type_name, key=len)} + self.typing_preamble = BuildPreamble(self.definitions) - self.typing_preamble = 'DO $$\nBEGIN\n' + '\n'.join(self.definitions) + '\nEND $$;\n' \ No newline at end of file +def BuildPreamble(definitions): + return 'DO $$\nBEGIN\n' + '\n'.join(definitions.values()) + '\nEND $$;\n' diff --git a/type_inference/research/reference_algebra.py b/type_inference/research/reference_algebra.py index 0891738f..25594974 100644 --- a/type_inference/research/reference_algebra.py +++ b/type_inference/research/reference_algebra.py @@ -71,8 +71,8 @@ def __str__(self): f'belongs to a list, but is implied to be {colored_t2}. ' f'Logica has to follow existing DB practice (Posgres, BigQuery) ' f'and disallow lists to be elements of lists. This includes ' - f'ArgMax and ArgMin aggregations, as they use SQL arrays as an ' - f'intermediate. Kindly wrap your inner list into a single field ' + f'ArgMaxK and ArgMinK aggregations, as they build lists. ' + f'Kindly wrap your inner list into a single field ' f'record.' ) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index cedfcdde..fbf864d1 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -83,12 +83,12 @@ def TypesOfBultins(): 'logica_value': reference_algebra.ClosedRecord({'arg': x, 'value': y}) }, 'ArgMin': { - 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), - 'logica_value': e + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 'logica_value': x }, 'ArgMax': { - 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), - 'logica_value': e + 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), + 'logica_value': x }, 'ArgMinK': { 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), From 17e8f255a06142e7b66460cc54aaf5da4ec3340c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Wed, 9 Aug 2023 00:43:25 +0000 Subject: [PATCH 113/141] Update psql ArgMin test. --- integration_tests/psql_argmin_list_test.txt | 5 +++++ integration_tests/run_tests.py | 1 + 2 files changed, 6 insertions(+) create mode 100644 integration_tests/psql_argmin_list_test.txt diff --git a/integration_tests/psql_argmin_list_test.txt b/integration_tests/psql_argmin_list_test.txt new file mode 100644 index 00000000..8dcddc7c --- /dev/null +++ b/integration_tests/psql_argmin_list_test.txt @@ -0,0 +1,5 @@ + col0 | col1 +------+-------- + {3} | {9,10} +(1 row) + diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 4b4009b6..471b35e7 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -91,6 +91,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_argmin_list_test") RunTest("psql_purchase_test") RunTest("psql_purchase2_test") RunTest("psql_purchase3_test") From 04568281ebba14fe67e9d125fc97922ac0731055 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 8 Aug 2023 19:06:25 -0700 Subject: [PATCH 114/141] Refining. --- type_inference/research/types_of_builtins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index fbf864d1..03d2676e 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -190,6 +190,11 @@ def TypesOfBultins(): 'AnyValue': { 0: x, 'logica_value': x + }, + 'Format': { + 0: 'Str', + 1: 'Any', 2: 'Any', 3: 'Any', 4: 'Any', 5: 'Any', 6: 'Any', + 'logica_value': 'Str' } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] From 192a415280b8eba527d30572b9212de91f65d55c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 8 Aug 2023 23:08:14 -0700 Subject: [PATCH 115/141] Refining. --- compiler/dialects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index 2d189991..ed5b00d0 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -150,7 +150,7 @@ def BuiltInFunctions(self): 'ToString': 'CAST(%s AS TEXT)', 'ToInt64': 'CAST(%s AS BIGINT)', 'Element': '({0})[{1} + 1]', - 'Size': 'ARRAY_LENGTH(%s, 1)', + 'Size': 'COALESCE(ARRAY_LENGTH({0}, 1), 0)', 'Count': 'COUNT(DISTINCT {0})', 'MagicalEntangle': '(CASE WHEN {1} = 0 THEN {0} ELSE NULL END)', 'ArrayConcat': '{0} || {1}' From ec506cd7088fec575c8e007f7cc136fd663f08b4 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Fri, 11 Aug 2023 23:00:26 -0700 Subject: [PATCH 116/141] Working towards elimination of race condition. Also specifying Element type. --- compiler/expr_translate.py | 4 ++-- type_inference/research/types_of_builtins.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 92265421..8d16d58c 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -202,8 +202,8 @@ def CamelCase(s): int(row['min_args']), float('inf') if row['has_repeated_args'] == '1' else int(row['max_args'])) - cls.BULK_FUNCTIONS = bulk_functions - cls.BULK_FUNCTIONS_ARITY_RANGE = bulk_functions_arity_range + cls.BULK_FUNCTIONS = bulk_functions + cls.BULK_FUNCTIONS_ARITY_RANGE = bulk_functions_arity_range def BuiltInFunctionArityRange(self, f): """Returns arity of the built-in function.""" diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 03d2676e..8c646135 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -195,6 +195,11 @@ def TypesOfBultins(): 0: 'Str', 1: 'Any', 2: 'Any', 3: 'Any', 4: 'Any', 5: 'Any', 6: 'Any', 'logica_value': 'Str' + }, + 'Element': { + 0: list_of_e, + 1: 'Num', + 'logica_value': e } } types_of_predicate['<'] = types_of_predicate['<='] = types_of_predicate['>='] = types_of_predicate['>'] From 393b2e4c808eb5295b6f50df92e06c3772b24d7e Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:42:42 -0700 Subject: [PATCH 117/141] Add `1` aggregating function. --- compiler/expr_translate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/expr_translate.py b/compiler/expr_translate.py index 8d16d58c..c9bb299a 100755 --- a/compiler/expr_translate.py +++ b/compiler/expr_translate.py @@ -39,6 +39,7 @@ class QL(object): 'ToUInt64': 'CAST(%s AS UINT64)', 'ToString': 'CAST(%s AS STRING)', # Aggregation. + '1': 'MIN(%s)', # Should be ANY_VALUE, but using MIN, so it works everywhere. 'Aggr': '%s', # Placeholder to use formulas for aggregation. 'Agg+': 'SUM(%s)', 'Agg++': 'ARRAY_CONCAT_AGG(%s)', From f416598e0c5b11e453e77741ba96db0b4b3491d7 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 26 Aug 2023 20:08:25 -0500 Subject: [PATCH 118/141] Adding type equality operator. --- compiler/dialect_libraries/bq_library.py | 1 + compiler/dialect_libraries/presto_library.py | 1 + compiler/dialect_libraries/psql_library.py | 1 + compiler/dialect_libraries/sqlite_library.py | 1 + compiler/dialect_libraries/trino_library.py | 1 + integration_tests/psql_explicit_typing_test.l | 43 +++++++++++++++++++ integration_tests/run_tests.py | 1 + parser_py/parse.py | 7 ++- type_inference/research/infer.py | 8 ++++ type_inference/research/types_of_builtins.py | 14 ++++-- 10 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 integration_tests/psql_explicit_typing_test.l diff --git a/compiler/dialect_libraries/bq_library.py b/compiler/dialect_libraries/bq_library.py index d6cc3546..6c794716 100644 --- a/compiler/dialect_libraries/bq_library.py +++ b/compiler/dialect_libraries/bq_library.py @@ -17,6 +17,7 @@ library = """ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; +`~`(left:, right:); # No action. Compiler unifies types. # All ORDER BY arguments are wrapped, to avoid confusion with # column index. diff --git a/compiler/dialect_libraries/presto_library.py b/compiler/dialect_libraries/presto_library.py index 90da11d4..c7015bc4 100644 --- a/compiler/dialect_libraries/presto_library.py +++ b/compiler/dialect_libraries/presto_library.py @@ -17,6 +17,7 @@ library = """ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; +`~`(left:, right:); # No action. Compiler unifies types. ArgMin(a) = SqlExpr("(ARRAY_AGG({arg} order by {value}))[1]", {arg: a.arg, value: a.value}); diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 312f3842..db52216b 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -17,6 +17,7 @@ library = """ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; +`~`(left:, right:); # No action. Compiler unifies types. ArgMin(a) = (SqlExpr("(ARRAY_AGG({arg} order by {value}))[1]", {arg: {argpod: a.arg}, value: a.value})).argpod; diff --git a/compiler/dialect_libraries/sqlite_library.py b/compiler/dialect_libraries/sqlite_library.py index 284a94d7..1a4f4eb0 100644 --- a/compiler/dialect_libraries/sqlite_library.py +++ b/compiler/dialect_libraries/sqlite_library.py @@ -17,6 +17,7 @@ library = """ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; +`~`(left:, right:); # No action. Compiler unifies types. Arrow(left, right) = arrow :- left == arrow.arg, diff --git a/compiler/dialect_libraries/trino_library.py b/compiler/dialect_libraries/trino_library.py index 90da11d4..c7015bc4 100644 --- a/compiler/dialect_libraries/trino_library.py +++ b/compiler/dialect_libraries/trino_library.py @@ -17,6 +17,7 @@ library = """ ->(left:, right:) = {arg: left, value: right}; `=`(left:, right:) = right :- left == right; +`~`(left:, right:); # No action. Compiler unifies types. ArgMin(a) = SqlExpr("(ARRAY_AGG({arg} order by {value}))[1]", {arg: a.arg, value: a.value}); diff --git a/integration_tests/psql_explicit_typing_test.l b/integration_tests/psql_explicit_typing_test.l new file mode 100644 index 00000000..36c2edf3 --- /dev/null +++ b/integration_tests/psql_explicit_typing_test.l @@ -0,0 +1,43 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@Engine("psql"); + +@Ground(T); +T(a: 1, b: "I"); +T(a: 2, b: "II"); +T(a: 3, b: "III"); + +@Ground(R); +R(f: {c: 5, d: [{e: "e"}]}, g: "g"); + +SaveT() += 1 :- T(); +SaveR() += 1 :- R(); + +@Ground(D, "logica_test.T"); +@Ground(RawE, "logica_test.R"); + +E(..r) :- + RawE(..r), + r ~ {f: {c: Num, d: [{e: Str}]}, g: Str}; + +@Ground(PrepareData); +PrepareData("done") += 1 :- SaveT() | SaveR(); + +Test({f:, g:}) Array= a -> {a:, b:} :- + PrepareData(), + D(a:, b:), + E(f:, g:), + a ~ Num, b ~ Str; \ No newline at end of file diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 471b35e7..77a1d663 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -91,6 +91,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_explicit_typing_test") RunTest("psql_argmin_list_test") RunTest("psql_purchase_test") RunTest("psql_purchase2_test") diff --git a/parser_py/parse.py b/parser_py/parse.py index 0ff92298..5806280c 100755 --- a/parser_py/parse.py +++ b/parser_py/parse.py @@ -579,7 +579,7 @@ def ParseLiteral(s): def ParseInfix(s, operators=None): """Parses an infix operator expression.""" operators = operators or [ - '||', '&&', '->', '==', '<=', '>=', '<', '>', '!=', '=', + '||', '&&', '->', '==', '<=', '>=', '<', '>', '!=', '=', '~', ' in ', ' is not ', ' is ', '++?', '++', '+', '-', '*', '/', '%', '^', '!'] unary_operators = ['-', '!'] @@ -598,6 +598,8 @@ def ParseInfix(s, operators=None): 'predicate_name': op, 'record': ParseRecordInternals(right) } + if op == '~' and not left: + return None # Negation is special. left_expr = ParseExpression(left) right_expr = ParseExpression(right) @@ -807,6 +809,9 @@ def ParseGenericCall(s, opening, closing): # Specialcasing `=` assignment operator for definition. if predicate == '`=`': predicate = '=' + # Specialcasing `~` type equality operator for definition. + if predicate == '`~`': + predicate = '~' return predicate, s[idx + 1: -1] diff --git a/type_inference/research/infer.py b/type_inference/research/infer.py index 571d755c..b6b3b543 100644 --- a/type_inference/research/infer.py +++ b/type_inference/research/infer.py @@ -408,6 +408,13 @@ def ActMindingRecordLiterals(self, node): node['type']['the_type'].CloseRecord() + def ActMindingTypingPredicateLiterals(self, node): + if 'type' in node and 'literal' in node and 'the_predicate' in node['literal']: + predicate_name = node['literal']['the_predicate']['predicate_name'] + if predicate_name in ['Str', 'Num', 'Bool']: + reference_algebra.Unify(node['type']['the_type'], + reference_algebra.TypeReference(predicate_name)) + def ActMindingListLiterals(self, node): if 'type' in node and 'literal' in node and 'the_list' in node['literal']: list_type = node['type']['the_type'] @@ -447,6 +454,7 @@ def ActMindingImplications(self, node): ) def IterateInference(self): + Walk(self.rule, self.ActMindingTypingPredicateLiterals) Walk(self.rule, self.ActMindingRecordLiterals) Walk(self.rule, self.ActUnifying) Walk(self.rule, self.ActUnderstandingSubscription) diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index 8c646135..65c501e1 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -22,6 +22,8 @@ def TypesOfBultins(): x = reference_algebra.TypeReference('Any') y = reference_algebra.TypeReference('Any') + # Special X that ends up singular in SQLite. + special_x = reference_algebra.TypeReference('Any') list_of_e = reference_algebra.TypeReference('Any') e = reference_algebra.TypeReference('Singular') reference_algebra.UnifyListElement(list_of_e, e) @@ -41,6 +43,10 @@ def TypesOfBultins(): 'right': x, 'logica_value': x }, + '~': { + 'left': x, + 'right': x + }, '++': { 'left': 'Str', 'right': 'Str', @@ -83,12 +89,12 @@ def TypesOfBultins(): 'logica_value': reference_algebra.ClosedRecord({'arg': x, 'value': y}) }, 'ArgMin': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), - 'logica_value': x + 0: reference_algebra.ClosedRecord({'arg': special_x, 'value': y}), + 'logica_value': special_x }, 'ArgMax': { - 0: reference_algebra.ClosedRecord({'arg': x, 'value': y}), - 'logica_value': x + 0: reference_algebra.ClosedRecord({'arg': special_x, 'value': y}), + 'logica_value': special_x }, 'ArgMinK': { 0: reference_algebra.ClosedRecord({'arg': e, 'value': y}), From a3edba96f338fa8d09b8c3befdebc1541f047575 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Sun, 27 Aug 2023 01:21:54 +0000 Subject: [PATCH 119/141] Updating tests. --- integration_tests/psql_explicit_typing_test.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 integration_tests/psql_explicit_typing_test.txt diff --git a/integration_tests/psql_explicit_typing_test.txt b/integration_tests/psql_explicit_typing_test.txt new file mode 100644 index 00000000..3e767415 --- /dev/null +++ b/integration_tests/psql_explicit_typing_test.txt @@ -0,0 +1,5 @@ + col0 | logica_value +---------------------+------------------------------ + ("(5,""{(e)}"")",g) | {"(1,I)","(2,II)","(3,III)"} +(1 row) + From 8747a9dae060915db972af4a7b3eaa82faa620e9 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsvo Date: Sun, 27 Aug 2023 01:22:20 +0000 Subject: [PATCH 120/141] Updating tests. --- integration_tests/psql_recursion_test.l | 2 +- integration_tests/psql_recursion_test.txt | 26 +++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integration_tests/psql_recursion_test.l b/integration_tests/psql_recursion_test.l index 211d0524..e5c365fc 100644 --- a/integration_tests/psql_recursion_test.l +++ b/integration_tests/psql_recursion_test.l @@ -27,7 +27,7 @@ ComponentOf(x) Min= y :- Distance(x, y); @OrderBy(Analyzed, "vertex"); Analyzed(vertex: x, component: ComponentOf(x), - distances? Array= (-y) -> [y, Distance(x, y)]) distinct; + distances? Array= (-y) -> {y:, d: Distance(x, y)}) distinct; Test := Analyzed(); diff --git a/integration_tests/psql_recursion_test.txt b/integration_tests/psql_recursion_test.txt index 41a1b056..3dad4331 100644 --- a/integration_tests/psql_recursion_test.txt +++ b/integration_tests/psql_recursion_test.txt @@ -1,15 +1,15 @@ - vertex | component | distances ---------+-----------+--------------------------------- - 1 | 1 | {{5,4},{4,3},{3,2},{2,1},{1,0}} - 2 | 1 | {{5,3},{4,2},{3,1},{2,0},{1,1}} - 3 | 1 | {{5,2},{4,1},{3,0},{2,1},{1,2}} - 4 | 1 | {{5,1},{4,0},{3,1},{2,2},{1,3}} - 5 | 1 | {{5,0},{4,1},{3,2},{2,3},{1,4}} - 6 | 6 | {{8,2},{7,1},{6,0}} - 7 | 6 | {{8,1},{7,0},{6,1}} - 8 | 6 | {{8,0},{7,1},{6,2}} - 9 | 9 | {{11,1},{10,1},{9,0}} - 10 | 9 | {{11,1},{10,0},{9,1}} - 11 | 9 | {{11,0},{10,1},{9,1}} + vertex | component | distances +--------+-----------+------------------------------------------- + 1 | 1 | {"(4,5)","(3,4)","(2,3)","(1,2)","(0,1)"} + 2 | 1 | {"(3,5)","(2,4)","(1,3)","(0,2)","(1,1)"} + 3 | 1 | {"(2,5)","(1,4)","(0,3)","(1,2)","(2,1)"} + 4 | 1 | {"(1,5)","(0,4)","(1,3)","(2,2)","(3,1)"} + 5 | 1 | {"(0,5)","(1,4)","(2,3)","(3,2)","(4,1)"} + 6 | 6 | {"(2,8)","(1,7)","(0,6)"} + 7 | 6 | {"(1,8)","(0,7)","(1,6)"} + 8 | 6 | {"(0,8)","(1,7)","(2,6)"} + 9 | 9 | {"(1,11)","(1,10)","(0,9)"} + 10 | 9 | {"(1,11)","(0,10)","(1,9)"} + 11 | 9 | {"(0,11)","(1,10)","(1,9)"} (11 rows) From 1a9c49652e9236a74cf7fb4169eda49615dccfd2 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 26 Aug 2023 23:46:43 -0500 Subject: [PATCH 121/141] Refining. --- colab_logica.py | 1 + 1 file changed, 1 insertion(+) diff --git a/colab_logica.py b/colab_logica.py index c5221da0..3a204bb1 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -378,6 +378,7 @@ def PostgresJumpStart(): # Connect to the database. import psycopg2 connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica') + connection.autocommit = True print('Connected.') global DEFAULT_ENGINE From f98bf538ecd7cdcd2b6b0bb9b330a7ca7f7e6e23 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 10:20:11 -0700 Subject: [PATCH 122/141] Make logica_home default schema for Postgres. --- compiler/universe.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/universe.py b/compiler/universe.py index 487cf229..416d0603 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -161,7 +161,7 @@ def Preamble(self): preamble += ( '-- Initializing PostgreSQL environment.\n' 'set client_min_messages to warning;\n' - 'create schema if not exists logica_test;\n\n') + 'create schema if not exists logica_home;\n\n') return preamble def BuildFlagValues(self): @@ -253,7 +253,11 @@ def OrderBy(self, predicate_name): return FieldValuesAsList(self.annotations['@OrderBy'][predicate_name]) def Dataset(self): - return self.ExtractSingleton('@Dataset', 'logica_test') + default_dataset = 'logica_test' + # This change is intended for all engines in the future. + if self.Engine() == 'psql': + default_dataset = 'logica_home' + return self.ExtractSingleton('@Dataset', default_dataset) def Engine(self): engine = self.ExtractSingleton('@Engine', self.default_engine) From b97de4e01d0bf56b82fb0e16743593c7483813fb Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 10:28:01 -0700 Subject: [PATCH 123/141] Make logica_home default schema for Postgres. --- integration_tests/psql_explicit_typing_test.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/psql_explicit_typing_test.l b/integration_tests/psql_explicit_typing_test.l index 36c2edf3..98d2384f 100644 --- a/integration_tests/psql_explicit_typing_test.l +++ b/integration_tests/psql_explicit_typing_test.l @@ -26,8 +26,8 @@ R(f: {c: 5, d: [{e: "e"}]}, g: "g"); SaveT() += 1 :- T(); SaveR() += 1 :- R(); -@Ground(D, "logica_test.T"); -@Ground(RawE, "logica_test.R"); +@Ground(D, "logica_home.T"); +@Ground(RawE, "logica_home.R"); E(..r) :- RawE(..r), From bdda1f0b463a057fa849f65540a1e5d1bfbf8ed9 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 16:19:59 -0700 Subject: [PATCH 124/141] Adding a test. --- integration_tests/psql_game_test.l | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 integration_tests/psql_game_test.l diff --git a/integration_tests/psql_game_test.l b/integration_tests/psql_game_test.l new file mode 100644 index 00000000..79823813 --- /dev/null +++ b/integration_tests/psql_game_test.l @@ -0,0 +1,73 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# What have we learned writing this test? +# Recursive errors need improvement. +# 1. If recursion fails to eliminate, it may manifest as psql type errors. +/* +@Recursive(Advantage, 5); +BestMove(x, + target? ArgMin= y -> Advantage(y), + value? Min= Advantage(y)) distinct :- + E(x, y); + +Advantage(x) += EndgamePrize(x); +Advantage(x) += -worst_value * 0.90 :- + V(x), + BestMove(x, value: worst_value); +*/ + +# 2. Recursion of a predicate on itself and on recursive fails to eliminate. +/* +@Recursive(BestMove, 5); +BestMove(x, + target? ArgMin= y -> Advantage(y), + value? Min= Advantage(y)) distinct :- + E(x, y); + +Advantage(x) += EndgamePrize(x); +Advantage(x) += -worst_value * 0.90 :- + Advantage(Endgame()), + V(x), + BestMove(x, value: worst_value); +*/ + + +@Engine("psql"); + +E(1, 2); +E(2, 1); +E(2, 3); +E(3, 4); +E(4, 5); +V(x) distinct :- x in [a,b], E(a,b); + +Endgame() = 5; +EndgamePrize(Endgame()) = 1; + +@Recursive(BestMove, 20); +@OrderBy(BestMove, "col0"); +BestMove(x, + target? ArgMin= y -> Advantage(y), + value? Min= Advantage(y)) distinct :- + E(x, y); + +Advantage(x) += EndgamePrize(x); +Advantage(x) += 0 :- V(x); +Advantage(x) += -worst_value * 0.90 :- + V(x), + BestMove(x, value: worst_value); + +Test := BestMove(); \ No newline at end of file From 56a73da8752e8d7e1b91ee287ff41cae6cbba601 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 16:32:57 -0700 Subject: [PATCH 125/141] Adding a test. --- integration_tests/psql_game_test.txt | 0 integration_tests/run_tests.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 integration_tests/psql_game_test.txt diff --git a/integration_tests/psql_game_test.txt b/integration_tests/psql_game_test.txt new file mode 100644 index 00000000..e69de29b diff --git a/integration_tests/run_tests.py b/integration_tests/run_tests.py index 77a1d663..75c869f5 100755 --- a/integration_tests/run_tests.py +++ b/integration_tests/run_tests.py @@ -91,6 +91,7 @@ def RunAll(test_presto=False, test_trino=False): RunTest("sqlite_reachability") RunTest("sqlite_element_test") + RunTest("psql_game_test") RunTest("psql_explicit_typing_test") RunTest("psql_argmin_list_test") RunTest("psql_purchase_test") From 331661f5026f9f241fc6ce91c9d0e4aacc137245 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 16:35:25 -0700 Subject: [PATCH 126/141] Adding a test. --- integration_tests/psql_game_test.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integration_tests/psql_game_test.txt b/integration_tests/psql_game_test.txt index e69de29b..f9b09fd7 100644 --- a/integration_tests/psql_game_test.txt +++ b/integration_tests/psql_game_test.txt @@ -0,0 +1,8 @@ + col0 | target | value +------+--------+-------------------------------------------- + 1 | 2 | 0.0000000000000000000000000000000000000000 + 2 | 1 | 0.0000000000000000000000000000000000000000 + 3 | 4 | -0.90 + 4 | 5 | 1 +(4 rows) + From 9a3d0ca5c4cd75543b3dd2416c36047b95743d3d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 16 Sep 2023 21:28:38 -0700 Subject: [PATCH 127/141] Refining. --- compiler/dialects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/dialects.py b/compiler/dialects.py index ed5b00d0..8b48ee3d 100755 --- a/compiler/dialects.py +++ b/compiler/dialects.py @@ -153,7 +153,8 @@ def BuiltInFunctions(self): 'Size': 'COALESCE(ARRAY_LENGTH({0}, 1), 0)', 'Count': 'COUNT(DISTINCT {0})', 'MagicalEntangle': '(CASE WHEN {1} = 0 THEN {0} ELSE NULL END)', - 'ArrayConcat': '{0} || {1}' + 'ArrayConcat': '{0} || {1}', + 'Split': 'STRING_TO_ARRAY({0}, {1})' } def InfixOperators(self): From 7b1d23834dd229082240ebb96e997bc9b64476c3 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 18 Sep 2023 10:09:11 -0700 Subject: [PATCH 128/141] Refining. --- common/concertina_lib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/concertina_lib.py b/common/concertina_lib.py index dc1b8166..b0a3c3ca 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -5,7 +5,10 @@ try: import graphviz except: - print('Could not import graphviz tools in Concertina.') + pass + # This is annoying to see in terminal each time. + # Consider adding back if lack of messaging is confusing. + # print('Could not import graphviz tools in Concertina.') try: from IPython.display import HTML From a88dd0b04851b9fe5efd7f83b3ddacc309d0bec6 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:55:15 -0700 Subject: [PATCH 129/141] Refining. --- common/concertina_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/concertina_lib.py b/common/concertina_lib.py index b0a3c3ca..8f9ac614 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -41,7 +41,7 @@ def Run(self, action): is_final=(predicate in self.final_predicates)) end = datetime.datetime.now() if self.print_running_predicate: - print(' (%d ms)' % ((end - start).microseconds / 1000)) + print(' (%d ms)' % int((end - start).total_seconds() * 1000)) if predicate in self.final_predicates: self.final_result[predicate] = result From b6ef12f7ffcca39c87dd340cc28076116fb2735f Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Thu, 21 Sep 2023 10:31:27 -0700 Subject: [PATCH 130/141] Allowing multiple preambles due to internal types. --- colab_logica.py | 3 +++ common/concertina_lib.py | 12 ++++++++---- compiler/universe.py | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 3a204bb1..13fee043 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -268,6 +268,9 @@ def Logica(line, cell, run_query): executions = [] sub_bars = [] ip = IPython.get_ipython() + for idx, predicate in enumerate(predicates): + # Prepopulating types. + for idx, predicate in enumerate(predicates): with bar.output_to(logs_idx): try: diff --git a/common/concertina_lib.py b/common/concertina_lib.py index 8f9ac614..e4ff2b4c 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -293,10 +293,14 @@ def ConcertinaConfig(table_to_export_map, dependency_edges, print_running_predicate=(display_mode != 'terminal')) preambles = set(e.preamble for e in logica_executions) - assert len(preambles) == 1, 'Inconsistent preambles: %s' % preambles - [preamble] = list(preambles) - if preamble: - sql_runner(preamble, sql_engine, is_final=False) + # Due to change of types from predicate to predicate preables are not + # consistent. However we expect preambles to be idempotent. + # So we simply run all of them. + # assert len(preambles) == 1, 'Inconsistent preambles: %s' % preambles + # [preamble] = list(preambles) + for preamble in preambles: + if preamble: + sql_runner(preamble, sql_engine, is_final=False) concertina = Concertina(config, engine, display_mode=display_mode) concertina.Run() diff --git a/compiler/universe.py b/compiler/universe.py index 416d0603..f29452cb 100755 --- a/compiler/universe.py +++ b/compiler/universe.py @@ -987,6 +987,8 @@ def SingleRuleSql(self, rule, s.UnificationsToConstraints() type_inference = infer.TypeInferenceForStructure(s, self.predicate_signatures) type_inference.PerformInference() + # New types may arrive here when we have an injetible predicate with variables + # which specific record type depends on the inputs. self.required_type_definitions.update(type_inference.collector.definitions) self.typing_preamble = infer.BuildPreamble(self.required_type_definitions) From 3078fe8bdca8e8208ba6e14a484353e2bb784440 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Thu, 21 Sep 2023 10:44:39 -0700 Subject: [PATCH 131/141] Allowing multiple preambles due to internal types. --- colab_logica.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 13fee043..3a204bb1 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -268,9 +268,6 @@ def Logica(line, cell, run_query): executions = [] sub_bars = [] ip = IPython.get_ipython() - for idx, predicate in enumerate(predicates): - # Prepopulating types. - for idx, predicate in enumerate(predicates): with bar.output_to(logs_idx): try: From 0fc99f423f87b8dc42ef051a5842925f08e5905d Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:38:02 -0700 Subject: [PATCH 132/141] Refining. --- colab_logica.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 3a204bb1..8e73e3e2 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -213,6 +213,14 @@ def __init__(self): global DB_CONNECTION global DB_ENGINE if not DB_CONNECTION: + print("Assuming this is running on Google CoLab in a temporary") + print("environment.") + print("Would you like to install and run postgres?") + user_choice = input('y or N? ') + if user_choice != 'y': + print('User declined.') + print('Bailing out.') + return PostgresJumpStart() self.connection = DB_CONNECTION @@ -330,14 +338,6 @@ def Logica(line, cell, run_query): print(' ') # To activate the tabbar. def PostgresJumpStart(): - print("Assuming this is running on Google CoLab in a temporary") - print("environment.") - print("Would you like to install and run postgres?") - user_choice = input('y or N? ') - if user_choice != 'y': - print('User declined.') - print('Bailing out.') - return # Install postgresql server. print("Installing and configuring an empty PostgreSQL database.") result = 0 From c82730a7089a8f25b4b97fc7051b00bb087c8bab Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:59:01 -0700 Subject: [PATCH 133/141] Refining. --- colab_logica.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/colab_logica.py b/colab_logica.py index 8e73e3e2..8efe885a 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -368,11 +368,12 @@ def PostgresJumpStart(): # Connect to the database. from logica import colab_logica -from sqlalchemy import create_engine -import pandas -engine = create_engine('postgresql+psycopg2://logica:logica@127.0.0.1', pool_recycle=3600); -connection = engine.connect(); -colab_logica.SetDbConnection(connection)""") +import psycopg2 +connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica') +connection.autocommit = True +colab_logica.DEFAULT_ENGINE = 'psql' +colab_logica.SetDbConnection(connection) +""") return print('Installation succeeded. Connecting...') # Connect to the database. From c393bd6f5b2fe903a4231b9e809a88b7943b55e8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 20:58:13 -0700 Subject: [PATCH 134/141] Update README.md --- README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d4d3913f..230e7e39 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,18 @@ a language created at Google earlier. ## Why? Logica is for engineers, data scientists and other specialists who want to use -logic programming syntax when writing queries and pipelines to run on -[BigQuery](https://cloud.google.com/bigquery). +logic programming syntax when writing queries and pipelines for databases and datawarehouses. +Logica programs run on +[BigQuery](https://cloud.google.com/bigquery), [Postgres](https://postgresql.org) and [SQLite](https://www.sqlite.org/). -Logica compiles to StandardSQL and gives you access to the power of BigQuery -engine with the convenience of logic programming syntax. This is useful because -BigQuery is magnitudes more powerful than state of the art native -logic programming engines. +Logica compiles to SQL and gives you access to the power of SQL ecosystem +with the convenience of logic programming syntax. + +This is useful because +SQL enginers are magnitudes more powerful than state of the art native +logic programming engines. For example, BigQuery is a distributed datawarehouse and thus logic programs written +in Logica can be easily parallelized onto thousands of servers. Postgres and SQLite are among most popular databases, they are +capable of processing substantial volumes of data right on your machine. We encourage you to try Logica, especially if @@ -38,7 +43,7 @@ We encourage you to try Logica, especially if * you use SQL, but feel unsatisfied about its readability, **or** * you want to learn logic programming and apply it to processing of Big Data. -In the future we plan to support more SQL dialects and engines. +Support more SQL dialects and engines is coming in the future. ## I have not heard of logic programming. What is it? From ed027a2d63c16eb5cea7e24bdd7f7387715da98c Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 22:47:43 -0700 Subject: [PATCH 135/141] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 230e7e39..8186ccb4 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ We encourage you to try Logica, especially if * you use SQL, but feel unsatisfied about its readability, **or** * you want to learn logic programming and apply it to processing of Big Data. -Support more SQL dialects and engines is coming in the future. +Support for more SQL dialects and engines is coming in the future. ## I have not heard of logic programming. What is it? From a884663f61b427adba4f39896c1370b7ba358940 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 22:54:30 -0700 Subject: [PATCH 136/141] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8186ccb4..785a73a1 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ capable of processing substantial volumes of data right on your machine. We encourage you to try Logica, especially if * you already use logic programming and need more computational power, **or** -* you use SQL, but feel unsatisfied about its readability, **or** +* you already have data in a SQL database/datawarehouse supported by Logica like BigQuery, PostgreSQL, SQLite, **or** * you want to learn logic programming and apply it to processing of Big Data. Support for more SQL dialects and engines is coming in the future. From f615c652e30be069f7546eb524a6c92c303888ee Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:06:21 -0700 Subject: [PATCH 137/141] Update index.html --- docs/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.html b/docs/index.html index 37ecd0b0..2f33e24a 100644 --- a/docs/index.html +++ b/docs/index.html @@ -347,7 +347,7 @@

Why Logica?

  • you already use logic programming and need more computational power, or
    • -
  • you use SQL, but feel unsatisfied about its readability, or
    • +
  • you already have data in a SQL database/datawarehouse supported by Logica like BigQuery, PostgreSQL, SQLite, or
  • you want to learn logic programming and apply it to processing of Big Data.
From ad49044ec85a802d5df71f74477dc23679f1bf61 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:09:08 -0700 Subject: [PATCH 138/141] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 785a73a1..257685ff 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ capable of processing substantial volumes of data right on your machine. We encourage you to try Logica, especially if * you already use logic programming and need more computational power, **or** -* you already have data in a SQL database/datawarehouse supported by Logica like BigQuery, PostgreSQL, SQLite, **or** +* you already have data in BigQuery, PostgreSQL or SQLite, **or** * you want to learn logic programming and apply it to processing of Big Data. Support for more SQL dialects and engines is coming in the future. From fa3d20e3602f0b7d45345b9634825854dc574961 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:10:09 -0700 Subject: [PATCH 139/141] Update index.html --- docs/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.html b/docs/index.html index 2f33e24a..872c1258 100644 --- a/docs/index.html +++ b/docs/index.html @@ -347,7 +347,7 @@

Why Logica?

  • you already use logic programming and need more computational power, or
    • -
  • you already have data in a SQL database/datawarehouse supported by Logica like BigQuery, PostgreSQL, SQLite, or
    • +
  • you already have data in BigQuery, PostgreSQL or SQLite, or
  • you want to learn logic programming and apply it to processing of Big Data.
From 501710acd2046b4d36a8820a9197d39dbbe9b185 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:12:11 -0700 Subject: [PATCH 140/141] Update index.html --- docs/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.html b/docs/index.html index 872c1258..d142326c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -329,7 +329,8 @@

Why Logica?

Logica is for engineers, data scientists and other specialists who need to perform complex data processing and analysis. Queries and pipelines written in Logica can run on -BigQuery and SQLite engines. +BigQuery, SQLite +and PostgreSQL engines. Information stored in these systems is thus available in Logica. From 872b27add8ff5a0896352c63b46cbeccf84cbdf9 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:14:56 -0700 Subject: [PATCH 141/141] Update index.html --- docs/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.html b/docs/index.html index d142326c..0242a23c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -352,7 +352,7 @@

Why Logica?

  • you want to learn logic programming and apply it to processing of Big Data.
    • -Among other engines, there is partial support for Trino and Postgres. +Among other engines, there is partial support for Trino and Databricks. Contributions to improve this support are very welcome!