From 6785a12aaae261d4edad28dfa39eddcef801482a Mon Sep 17 00:00:00 2001 From: Arthur Evans Date: Thu, 28 Mar 2024 10:07:53 -0700 Subject: [PATCH 01/55] [Docs] Update load()/store() item and move it to top of removed section. (#36110) modular-orig-commit: 038e34dca15cff2a2bdb22285f6d90628737a66d --- docs/changelog-released.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/changelog-released.md b/docs/changelog-released.md index bdd2bc5bbf..0b83b56536 100644 --- a/docs/changelog-released.md +++ b/docs/changelog-released.md @@ -224,6 +224,25 @@ modular update mojo #### πŸ¦‹ Changed +- The `simd_load()`, `simd_store()`, `aligned_simd_load()`, and + `aligned_simd_store()` methods on + [`DTypePointer`](/mojo/stdlib/memory/unsafe#dtypepointer), + [`Buffer`](/mojo/stdlib/buffer/buffer#buffer), and + [`NDBuffer`](/mojo/stdlib/buffer/buffer#ndbuffer) have been merged into + a more expressive set of `load()` and `store()` methods with keyword-only + `width` and `alignment` parameters: + + ```mojo + # Doesn't work + my_simd = my_buffer.simd_load[simd_width](index) + # Works + my_simd = my_buffer.load[width=simd_width](index) + # Doesn't work + my_buffer.aligned_simd_store[width, alignment](my_simd) + # Works + my_buffer.store[width=width, alignment=alignment](my_simd) + ``` + - The [`EqualityComparable`](/mojo/stdlib/builtin/equality_comparable#equalitycomparable) trait now requires the `__ne__()` method for conformance in addition to the @@ -379,19 +398,6 @@ modular update mojo memcpy(destBuffer.data, srcBuffer.data, count) ``` -- The `simd_load()` and `simd_store()` methods on - [`DTypePointer`](/mojo/stdlib/memory/unsafe#dtypepointer), - [`Buffer`](/mojo/stdlib/buffer/buffer#buffer), and - [`NDBuffer`](/mojo/stdlib/buffer/buffer#ndbuffer) have been removed in favor - of `load()` and `store()`: - - ```mojo - # Doesn't work - my_simd = my_buffer.simd_load[simd_width](index) - # Works - my_simd = my_buffer.load[simd_width, alignment](index) - ``` - - The functions `max_or_inf()`, `min_or_neginf()` have been removed from `math.limit`. These functions were only used by the SIMD type. From 2267591e43446a99c6749be8351a73bd7f9c023d Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Thu, 28 Mar 2024 14:10:06 -0700 Subject: [PATCH 02/55] [mojo-stdlib] Move a few more regpassable inits to 'inout self'. (#36150) This just cleans up a few more of these in builtin_list. modular-orig-commit: 0f88978e8005a0e0aae4c2e279b57cbca8b101c7 --- stdlib/src/builtin/builtin_list.mojo | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 200e79d97b..f37890a35a 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -36,16 +36,13 @@ struct ListLiteral[*Ts: AnyRegType](Sized): """The underlying storage for the list.""" @always_inline("nodebug") - fn __init__(*args: *Ts) -> Self: + fn __init__(inout self, *args: *Ts): """Construct the list literal from the given values. Args: args: The init values. - - Returns: - The constructed ListLiteral. """ - return Self {storage: args} + self.storage = args @always_inline("nodebug") fn __len__(self) -> Int: @@ -113,29 +110,23 @@ struct VariadicList[type: AnyRegType](Sized): alias IterType = _VariadicListIter[type] @always_inline - fn __init__(*value: type) -> Self: + fn __init__(inout self, *value: type): """Constructs a VariadicList from a variadic list of arguments. Args: value: The variadic argument list to construct the variadic list with. - - Returns: - The VariadicList constructed. """ - return value + self = value @always_inline - fn __init__(value: Self.storage_type) -> Self: + fn __init__(inout self, value: Self.storage_type): """Constructs a VariadicList from a variadic argument type. Args: value: The variadic argument to construct the list with. - - Returns: - The VariadicList constructed. """ - return Self {value: value} + self.value = value @always_inline fn __len__(self) -> Int: From 6c943c90deb3d2a9188f8accc0ae7c368651701b Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Thu, 28 Mar 2024 15:20:32 -0700 Subject: [PATCH 03/55] [mojo-lang] Change mlir-ification of values of trait type. (#36166) This changes substituteMLIRMagic to include the trait type in the expanded string for a value of trait type in a KGEN type context. This makes it easier and less error prone to work with trait values. If there is a reason to get the raw type without a preceding type, then you can use the unary plus hack like for other attribute values. modular-orig-commit: 09c8a037b73d016d139e8391bb37d31c943ed9b4 --- stdlib/src/memory/anypointer.mojo | 2 +- stdlib/src/memory/unsafe.mojo | 8 ++------ stdlib/src/utils/variant.mojo | 4 +--- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/stdlib/src/memory/anypointer.mojo b/stdlib/src/memory/anypointer.mojo index ba23dc84c3..251f1275a4 100644 --- a/stdlib/src/memory/anypointer.mojo +++ b/stdlib/src/memory/anypointer.mojo @@ -36,7 +36,7 @@ struct AnyPointer[T: Movable]( T: The pointer element type, which must be movable. """ - alias pointer_type = __mlir_type[`!kgen.pointer<:`, Movable, ` `, T, `>`] + alias pointer_type = __mlir_type[`!kgen.pointer<`, T, `>`] """The underlying pointer type.""" var value: Self.pointer_type """The underlying pointer.""" diff --git a/stdlib/src/memory/unsafe.mojo b/stdlib/src/memory/unsafe.mojo index 817ae838b5..c5a0540482 100644 --- a/stdlib/src/memory/unsafe.mojo +++ b/stdlib/src/memory/unsafe.mojo @@ -435,9 +435,7 @@ struct _LITRef[ addr_space: __mlir_type.index = Int(0).__mlir_index__(), ]: alias type = __mlir_type[ - `!lit.ref<:`, - AnyType, - ` `, + `!lit.ref<`, element_type, `, `, lifetime, @@ -538,9 +536,7 @@ struct Reference[ # to KGEN pointer. var kgen_ptr = __mlir_op.`lit.ref.to_pointer`(self.value) var dest_ptr = __mlir_op.`pop.pointer.bitcast`[ - _type = __mlir_type[ - `!kgen.pointer<:`, AnyType, ` `, new_element_type, `>` - ] + _type = __mlir_type[`!kgen.pointer<`, new_element_type, `>`] ](kgen_ptr) return __mlir_op.`lit.ref.from_pointer`[ _type = _LITRef[new_element_type, is_mutable, lifetime].type diff --git a/stdlib/src/utils/variant.mojo b/stdlib/src/utils/variant.mojo index 4677f12ae1..66c473cfab 100644 --- a/stdlib/src/utils/variant.mojo +++ b/stdlib/src/utils/variant.mojo @@ -154,9 +154,7 @@ struct Variant[*Ts: CollectionElement](CollectionElement): var ptr = Reference(self._impl).get_unsafe_pointer().address var result = AnyPointer[T]() result.value = __mlir_op.`pop.pointer.bitcast`[ - _type = __mlir_type[ - `!kgen.pointer<:`, CollectionElement, ` `, T, `>` - ] + _type = __mlir_type[`!kgen.pointer<`, T, `>`] ](ptr) return result From b6fffb287021010f0956955157c7c964c3cbda0a Mon Sep 17 00:00:00 2001 From: River Riddle Date: Thu, 28 Mar 2024 18:01:22 -0600 Subject: [PATCH 04/55] [mojo] Error out on defining `main` in a package (#36062) The semantics of this are not well defined for mojo right now, so it's cleaner to just disallow it. We can loosen this up when we have cleaner semantics around entry points in packages. Closes [Internal Link] modular-orig-commit: 3203283b41bfd984cd183ba1a314c0c3ba13925d --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 1c1d94a945..c7854c4ce9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -32,3 +32,7 @@ and tools. Please add any significant user-visible changes here. ### ❌ Removed ### πŸ› οΈ Fixed + +- [#1987](https://github.com/modularml/mojo/issues/1987) Defining `main` + in a Mojo package is an error, for now. This is not intended to work yet, + erroring for now will help to prevent accidental undefined behavior. From fe36e1b88a000b0e55248a8c507b4ccb28163bf3 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Thu, 28 Mar 2024 18:02:52 -0600 Subject: [PATCH 05/55] [mojo-stdlib] Remove always_inline from debug_assert (#36065) This was required originally because of issues related to packaging, but with the removal of separate package codegen, this is no longer necessary. Closes #24957 modular-orig-commit: 79be2c2c74f2aefac95625115a8b7baf291c25ad --- stdlib/src/builtin/debug_assert.mojo | 3 --- 1 file changed, 3 deletions(-) diff --git a/stdlib/src/builtin/debug_assert.mojo b/stdlib/src/builtin/debug_assert.mojo index dd4ca9b885..454bf42c64 100644 --- a/stdlib/src/builtin/debug_assert.mojo +++ b/stdlib/src/builtin/debug_assert.mojo @@ -22,7 +22,6 @@ from sys.info import triple_is_nvidia_cuda from sys.param_env import is_defined -@always_inline fn debug_assert(cond: Bool, msg: StringLiteral): """Asserts that the condition is true. @@ -40,7 +39,6 @@ fn debug_assert(cond: Bool, msg: StringLiteral): _debug_assert_impl(cond, msg) -@always_inline fn debug_assert[boolable: Boolable](cond: boolable, msg: StringLiteral): """Asserts that the condition is true. @@ -61,7 +59,6 @@ fn debug_assert[boolable: Boolable](cond: boolable, msg: StringLiteral): _debug_assert_impl(cond, msg) -@always_inline fn _debug_assert_impl[boolable: Boolable](cond: boolable, msg: StringLiteral): """Asserts that the condition is true.""" From 39207867d8bfadcf62406b6c195bc7f1258c6503 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Thu, 28 Mar 2024 18:09:11 -0600 Subject: [PATCH 06/55] [mojo-driver] Add a `-g` option that aliases `--debug-level full` (#36069) This provides a more familiar way to enable full debugging that matches many other languages. Closes #24171 modular-orig-commit: 7d9038b935bbf86b775237a33b0cb294798200af --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index c7854c4ce9..d4b11565ad 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -27,6 +27,10 @@ and tools. Please add any significant user-visible changes here. `2x2` tensor which is initialized with all zeros. This provides an easy way to fill the data of a tensor. +- The `mojo build` and `mojo run` commands now support a `-g` option. This + shorter alias is equivalent to writing `--debug-level full`. This option is + also available in the `mojo debug` command, but is already the default. + ### πŸ¦‹ Changed ### ❌ Removed From 514c99d781a8e98eff1ccb835f7ab58b25f6d27e Mon Sep 17 00:00:00 2001 From: abdul dakkak Date: Fri, 29 Mar 2024 07:00:27 -0700 Subject: [PATCH 07/55] [Stdlib] Reconcile the div_ceil and ceildiv functions (#36203) We only need a single one of these, so just remove the div_ceil function in place of the ceildiv one (since the ceildiv is what it's called in Python). Closes #33323 modular-orig-commit: 627e5eef1aa20d7c03f0330ad608b2dbbe24a945 --- docs/changelog.md | 4 ++++ stdlib/src/builtin/hash.mojo | 4 ++-- stdlib/src/builtin/io.mojo | 4 ++-- stdlib/src/builtin/range.mojo | 4 ++-- stdlib/src/builtin/tuple.mojo | 4 ++-- stdlib/src/utils/stringref.mojo | 4 ++-- stdlib/src/utils/variant.mojo | 4 ++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d4b11565ad..dad07f2a45 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,6 +31,10 @@ and tools. Please add any significant user-visible changes here. shorter alias is equivalent to writing `--debug-level full`. This option is also available in the `mojo debug` command, but is already the default. +- Due to an oversight there was a `ceildiv` and `div_ceil` function in the + `math` module. These two functions have been reconciled with the `div_ceil` + being removed. + ### πŸ¦‹ Changed ### ❌ Removed diff --git a/stdlib/src/builtin/hash.mojo b/stdlib/src/builtin/hash.mojo index de4624a9fc..c5db156fff 100644 --- a/stdlib/src/builtin/hash.mojo +++ b/stdlib/src/builtin/hash.mojo @@ -36,7 +36,7 @@ from memory import memcpy, memset_zero, stack_allocation @always_inline -fn _div_ceil_positive(numerator: Int, denominator: Int) -> Int: +fn _ceildiv_positive(numerator: Int, denominator: Int) -> Int: return (numerator + denominator - 1)._positive_div(denominator) @@ -149,7 +149,7 @@ fn _hash_simd[type: DType, size: Int](data: SIMD[type, size]) -> Int: hash collision statistical properties for common data structures. """ # Some types will have non-integer ratios, eg. DType.bool - alias int8_size = _div_ceil_positive( + alias int8_size = _ceildiv_positive( type.bitwidth(), DType.uint8.bitwidth() ) * size # Stack allocate bytes for `data` and load it into that memory. diff --git a/stdlib/src/builtin/io.mojo b/stdlib/src/builtin/io.mojo index f8602a4810..ca95071121 100644 --- a/stdlib/src/builtin/io.mojo +++ b/stdlib/src/builtin/io.mojo @@ -30,8 +30,8 @@ from utils import StringRef, unroll @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var div_ceil = (value + alignment - 1)._positive_div(alignment) - return div_ceil * alignment + var ceildiv = (value + alignment - 1)._positive_div(alignment) + return ceildiv * alignment # ===----------------------------------------------------------------------=== # diff --git a/stdlib/src/builtin/range.mojo b/stdlib/src/builtin/range.mojo index 1d49d688eb..f4c817916d 100644 --- a/stdlib/src/builtin/range.mojo +++ b/stdlib/src/builtin/range.mojo @@ -24,7 +24,7 @@ from python.object import PythonObject @always_inline -fn _div_ceil_positive(numerator: Int, denominator: Int) -> Int: +fn _ceildiv_positive(numerator: Int, denominator: Int) -> Int: """Divides an integer by another integer, and round up to the nearest integer. @@ -161,7 +161,7 @@ struct _StridedRange(Sized): @always_inline("nodebug") fn __len__(self) -> Int: - return _div_ceil_positive(_abs(self.start - self.end), _abs(self.step)) + return _ceildiv_positive(_abs(self.start - self.end), _abs(self.step)) @always_inline("nodebug") fn __getitem__(self, idx: Int) -> Int: diff --git a/stdlib/src/builtin/tuple.mojo b/stdlib/src/builtin/tuple.mojo index 1c94d58ad4..dd97e4779e 100644 --- a/stdlib/src/builtin/tuple.mojo +++ b/stdlib/src/builtin/tuple.mojo @@ -103,5 +103,5 @@ struct Tuple[*Ts: AnyRegType](Sized, CollectionElement): @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var div_ceil = (value + alignment - 1)._positive_div(alignment) - return div_ceil * alignment + var ceildiv = (value + alignment - 1)._positive_div(alignment) + return ceildiv * alignment diff --git a/stdlib/src/utils/stringref.mojo b/stdlib/src/utils/stringref.mojo index 0fda1fa47e..bb9136cb50 100644 --- a/stdlib/src/utils/stringref.mojo +++ b/stdlib/src/utils/stringref.mojo @@ -25,8 +25,8 @@ from memory.unsafe import DTypePointer, Pointer @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var div_ceil = (value + alignment - 1)._positive_div(alignment) - return div_ceil * alignment + var ceildiv = (value + alignment - 1)._positive_div(alignment) + return ceildiv * alignment @always_inline diff --git a/stdlib/src/utils/variant.mojo b/stdlib/src/utils/variant.mojo index 66c473cfab..0a657c7b0e 100644 --- a/stdlib/src/utils/variant.mojo +++ b/stdlib/src/utils/variant.mojo @@ -53,8 +53,8 @@ from utils.static_tuple import StaticTuple @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var div_ceil = (value + alignment - 1)._positive_div(alignment) - return div_ceil * alignment + var ceildiv = (value + alignment - 1)._positive_div(alignment) + return ceildiv * alignment @always_inline From 07b594ea9fbd52714c4bff1c2eb0b6e4687f31e6 Mon Sep 17 00:00:00 2001 From: Jack Clayton Date: Fri, 29 Mar 2024 09:23:44 -0500 Subject: [PATCH 08/55] [mojo-examples] Fix incorrect logic in vectorize loop (#36196) The logic was incorrect if there was a remainder from C.cols % nelts modular-orig-commit: 347157c2401cdacd24f97f1a42027245438e168d --- examples/notebooks/Matmul.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/Matmul.ipynb b/examples/notebooks/Matmul.ipynb index 12483029b7..eff5e1f81d 100644 --- a/examples/notebooks/Matmul.ipynb +++ b/examples/notebooks/Matmul.ipynb @@ -589,7 +589,7 @@ "fn matmul_vectorized_0(C: Matrix, A: Matrix, B: Matrix):\n", " for m in range(C.rows):\n", " for k in range(A.cols):\n", - " for nv in range(0, C.cols, nelts):\n", + " for nv in range(0, C.cols - nelts + 1, nelts):\n", " C.store(m, nv, C.load[nelts](m, nv) + A[m, k] * B.load[nelts](k, nv))\n", "\n", " # Handle remaining elements with scalars.\n", From 8c545e649f3ec26a4892f632408061cb52c40eae Mon Sep 17 00:00:00 2001 From: Brian Gesiak Date: Fri, 29 Mar 2024 11:51:58 -0400 Subject: [PATCH 09/55] [CHANGELOG] Update for `mojo build` change (#36210) Update the changelog to reflect #36207, since it represents a slight change in the `mojo` tool's behavior. modular-orig-commit: 0e954f4d4768c3674dfd4f8989f67050bec140cf --- docs/changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index dad07f2a45..89683816a2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -37,6 +37,11 @@ and tools. Please add any significant user-visible changes here. ### πŸ¦‹ Changed +- The behavior of `mojo build` when invoked without an output `-o` argument has + changed slightly: `mojo build ./test-dir/program.mojo` now outputs an + executable to the path `./program`, whereas before it would output to the path + `./test-dir/program`. + ### ❌ Removed ### πŸ› οΈ Fixed From b3655820b0367207b22f319898ed74be03323b3b Mon Sep 17 00:00:00 2001 From: Scott Main Date: Fri, 29 Mar 2024 11:05:18 -0700 Subject: [PATCH 10/55] [docs] Cross-link the vision and roadmap docs (#36142) Maybe that wasn't the doc you were looking for. modular-orig-commit: fb7d62c22be4e0edf05c49e4624c9fa4a1f880f8 --- stdlib/docs/roadmap.md | 2 ++ stdlib/docs/vision.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/stdlib/docs/roadmap.md b/stdlib/docs/roadmap.md index 3ff96a1c24..7e05f86f0d 100644 --- a/stdlib/docs/roadmap.md +++ b/stdlib/docs/roadmap.md @@ -7,6 +7,8 @@ Modular's internal workflows. The roadmap updates act as a forcing function for discussions with the Mojo community to ensure the standard library contributors both internally and externally are aligned on the future technical direction. +For more about our long-term aspirations, check out our [Vision doc](vision.md). + ## 2024 Q2+ roadmap The following are high-level themes the Mojo standard library team will be diff --git a/stdlib/docs/vision.md b/stdlib/docs/vision.md index a015be8c79..b335e239af 100644 --- a/stdlib/docs/vision.md +++ b/stdlib/docs/vision.md @@ -3,6 +3,8 @@ This page outlines the principles we aspire to follow and the more concrete objectives and goals we aim to accomplish in the Mojo standard library. +For details about our near-term objectives, see the [Roadmap doc](roadmap.md). + ## Principles The following are β€œNorth Star” principles for the Mojo standard library. From bd9f474fbc73e6da576ebf7f12da85b1df9ffc90 Mon Sep 17 00:00:00 2001 From: abdul dakkak Date: Fri, 29 Mar 2024 15:08:11 -0700 Subject: [PATCH 11/55] [Internal Change] modular-orig-commit: 44d3a3560edfbd4510e7a906d895a5b01c04b606 --- docs/changelog.md | 4 ---- stdlib/src/builtin/hash.mojo | 4 ++-- stdlib/src/builtin/io.mojo | 4 ++-- stdlib/src/builtin/range.mojo | 4 ++-- stdlib/src/builtin/tuple.mojo | 4 ++-- stdlib/src/utils/stringref.mojo | 4 ++-- stdlib/src/utils/variant.mojo | 4 ++-- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 89683816a2..a0033f6c9b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,10 +31,6 @@ and tools. Please add any significant user-visible changes here. shorter alias is equivalent to writing `--debug-level full`. This option is also available in the `mojo debug` command, but is already the default. -- Due to an oversight there was a `ceildiv` and `div_ceil` function in the - `math` module. These two functions have been reconciled with the `div_ceil` - being removed. - ### πŸ¦‹ Changed - The behavior of `mojo build` when invoked without an output `-o` argument has diff --git a/stdlib/src/builtin/hash.mojo b/stdlib/src/builtin/hash.mojo index c5db156fff..de4624a9fc 100644 --- a/stdlib/src/builtin/hash.mojo +++ b/stdlib/src/builtin/hash.mojo @@ -36,7 +36,7 @@ from memory import memcpy, memset_zero, stack_allocation @always_inline -fn _ceildiv_positive(numerator: Int, denominator: Int) -> Int: +fn _div_ceil_positive(numerator: Int, denominator: Int) -> Int: return (numerator + denominator - 1)._positive_div(denominator) @@ -149,7 +149,7 @@ fn _hash_simd[type: DType, size: Int](data: SIMD[type, size]) -> Int: hash collision statistical properties for common data structures. """ # Some types will have non-integer ratios, eg. DType.bool - alias int8_size = _ceildiv_positive( + alias int8_size = _div_ceil_positive( type.bitwidth(), DType.uint8.bitwidth() ) * size # Stack allocate bytes for `data` and load it into that memory. diff --git a/stdlib/src/builtin/io.mojo b/stdlib/src/builtin/io.mojo index ca95071121..f8602a4810 100644 --- a/stdlib/src/builtin/io.mojo +++ b/stdlib/src/builtin/io.mojo @@ -30,8 +30,8 @@ from utils import StringRef, unroll @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var ceildiv = (value + alignment - 1)._positive_div(alignment) - return ceildiv * alignment + var div_ceil = (value + alignment - 1)._positive_div(alignment) + return div_ceil * alignment # ===----------------------------------------------------------------------=== # diff --git a/stdlib/src/builtin/range.mojo b/stdlib/src/builtin/range.mojo index f4c817916d..1d49d688eb 100644 --- a/stdlib/src/builtin/range.mojo +++ b/stdlib/src/builtin/range.mojo @@ -24,7 +24,7 @@ from python.object import PythonObject @always_inline -fn _ceildiv_positive(numerator: Int, denominator: Int) -> Int: +fn _div_ceil_positive(numerator: Int, denominator: Int) -> Int: """Divides an integer by another integer, and round up to the nearest integer. @@ -161,7 +161,7 @@ struct _StridedRange(Sized): @always_inline("nodebug") fn __len__(self) -> Int: - return _ceildiv_positive(_abs(self.start - self.end), _abs(self.step)) + return _div_ceil_positive(_abs(self.start - self.end), _abs(self.step)) @always_inline("nodebug") fn __getitem__(self, idx: Int) -> Int: diff --git a/stdlib/src/builtin/tuple.mojo b/stdlib/src/builtin/tuple.mojo index dd97e4779e..1c94d58ad4 100644 --- a/stdlib/src/builtin/tuple.mojo +++ b/stdlib/src/builtin/tuple.mojo @@ -103,5 +103,5 @@ struct Tuple[*Ts: AnyRegType](Sized, CollectionElement): @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var ceildiv = (value + alignment - 1)._positive_div(alignment) - return ceildiv * alignment + var div_ceil = (value + alignment - 1)._positive_div(alignment) + return div_ceil * alignment diff --git a/stdlib/src/utils/stringref.mojo b/stdlib/src/utils/stringref.mojo index bb9136cb50..0fda1fa47e 100644 --- a/stdlib/src/utils/stringref.mojo +++ b/stdlib/src/utils/stringref.mojo @@ -25,8 +25,8 @@ from memory.unsafe import DTypePointer, Pointer @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var ceildiv = (value + alignment - 1)._positive_div(alignment) - return ceildiv * alignment + var div_ceil = (value + alignment - 1)._positive_div(alignment) + return div_ceil * alignment @always_inline diff --git a/stdlib/src/utils/variant.mojo b/stdlib/src/utils/variant.mojo index 0a657c7b0e..66c473cfab 100644 --- a/stdlib/src/utils/variant.mojo +++ b/stdlib/src/utils/variant.mojo @@ -53,8 +53,8 @@ from utils.static_tuple import StaticTuple @always_inline fn _align_up(value: Int, alignment: Int) -> Int: - var ceildiv = (value + alignment - 1)._positive_div(alignment) - return ceildiv * alignment + var div_ceil = (value + alignment - 1)._positive_div(alignment) + return div_ceil * alignment @always_inline From b6df7e70db8156d3cfc4c275c4d9a0c6bd0cba1d Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Fri, 29 Mar 2024 15:59:56 -0700 Subject: [PATCH 12/55] [mojo-stdlib] Move DTypePointer inits to `inout Self` (#36262) This is split off of #33595 modular-orig-commit: d745e5673d768ab862661f0ada2adb91f34e81e2 --- stdlib/src/memory/unsafe.mojo | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/stdlib/src/memory/unsafe.mojo b/stdlib/src/memory/unsafe.mojo index c5a0540482..ccd05eab31 100644 --- a/stdlib/src/memory/unsafe.mojo +++ b/stdlib/src/memory/unsafe.mojo @@ -1061,63 +1061,51 @@ struct DTypePointer[ """The pointed-to address.""" @always_inline("nodebug") - fn __init__() -> Self: - """Constructs a null `DTypePointer` from the given type. - - Returns: - Constructed `DTypePointer` object. - """ + fn __init__(inout self): + """Constructs a null `DTypePointer` from the given type.""" - return Self {address: Self.pointer_type()} + self.address = Self.pointer_type() @always_inline("nodebug") fn __init__( + inout self, value: __mlir_type[ `!kgen.pointer,`, address_space.value().value, `>`, - ] - ) -> Self: + ], + ): """Constructs a `DTypePointer` from a scalar pointer of the same type. Args: value: The scalar pointer. - - Returns: - Constructed `DTypePointer`. """ - return Pointer[ + self = Pointer[ __mlir_type[`!pop.scalar<`, type.value, `>`], address_space ](value).bitcast[Scalar[type]]() @always_inline("nodebug") - fn __init__(value: Pointer[Scalar[type], address_space]) -> Self: + fn __init__(inout self, value: Pointer[Scalar[type], address_space]): """Constructs a `DTypePointer` from a scalar pointer of the same type. Args: value: The scalar pointer. - - Returns: - Constructed `DTypePointer`. """ - return Self {address: value} + self.address = value @always_inline("nodebug") - fn __init__(value: Scalar[DType.address]) -> Self: + fn __init__(inout self, value: Scalar[DType.address]): """Constructs a `DTypePointer` from the value of scalar address. Args: value: The input pointer index. - - Returns: - Constructed `DTypePointer` object. """ var address = __mlir_op.`pop.index_to_pointer`[ _type = Self.pointer_type.pointer_type ](value.cast[DType.index]().value) - return Self {address: address} + self.address = address @staticmethod @always_inline("nodebug") From 53b1c9783daa8e7f956de9ffc82a5200ed5de8a7 Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Sat, 30 Mar 2024 11:03:07 -0700 Subject: [PATCH 13/55] [mojo][repl] Fix erroneous destructor call emission in generated REPL source (#35595) The REPL is doing dirty tricks with variables, turning them into heap allocations and tracking them in indirect structs. However, CheckLifetimes is onto its tricks, which causes it to reject these as incorrect values. The right solution is to make more invasive changes to the REPL, but that isn't in the short term plans. Workaround this by introducing a horrible hack (tm) op `RefFromPointerUntrackedOp` that creates a reference that isn't tracked by CheckLifetimes. This pokes a hole in our nice reference system which makes me very sad. A comment saying "don't use this" will prevent other people from using this for other things ... right??? --------- Co-authored-by: Chris Lattner modular-orig-commit: 7a6b7339522d2657f72d7e8a3a5b51e4d299167f --- docs/changelog.md | 6 ++++++ examples/notebooks/BoolMLIR.ipynb | 3 ++- examples/notebooks/HelloMojo.ipynb | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index a0033f6c9b..b6d02e059d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -37,6 +37,12 @@ and tools. Please add any significant user-visible changes here. changed slightly: `mojo build ./test-dir/program.mojo` now outputs an executable to the path `./program`, whereas before it would output to the path `./test-dir/program`. +- The REPL no longer allows type level variable declarations to be + uninitialized, e.g. it will reject `var s: String`. This is because it does + not do proper lifetime tracking (yet!) across cells, and so such code would + lead to a crash. You can work around this by initializing to a dummy value + and overwriting later. This limitation only applies to top level variables, + variables in functions work as they always have. ### ❌ Removed diff --git a/examples/notebooks/BoolMLIR.ipynb b/examples/notebooks/BoolMLIR.ipynb index a541c44c36..91613f5fe5 100644 --- a/examples/notebooks/BoolMLIR.ipynb +++ b/examples/notebooks/BoolMLIR.ipynb @@ -105,7 +105,8 @@ "metadata": {}, "outputs": [], "source": [ - "var a: OurBool" + "fn uninitialized_our_bool():\n", + " var a: OurBool\n" ] }, { diff --git a/examples/notebooks/HelloMojo.ipynb b/examples/notebooks/HelloMojo.ipynb index a60f594dd6..0159f99c22 100644 --- a/examples/notebooks/HelloMojo.ipynb +++ b/examples/notebooks/HelloMojo.ipynb @@ -65,7 +65,6 @@ } ], "source": [ - "#| XFAIL: *\n", "#| CHECK: Hello Mojo!\n", "print(\"Hello Mojo!\")" ] From d26d0b7e0683790cad8051fca203fffaea80c83a Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Sun, 31 Mar 2024 14:04:25 -0700 Subject: [PATCH 14/55] [mojo-lang] Expand `VariadicPack` to take parametric trait base (#36319) This patch is a big step towards enabling VariadicPack to model packs with a non-AnyType bound (e.g. all members must be Stringable). It expands `VariadicPack` with a new `element_trait` member that indicates what all the elements are, and teaches the parser to pass down the element type. The magic enabling this is support for metatype bound type expressions which is almost working, but not quite there yet. As such, this is another step, but isn't enough to declare success, which is why the integration test isn't using it yet. :-/ modular-orig-commit: d45b597eef0f58fc20770c2b7ec860216e9ecc6c --- stdlib/src/builtin/builtin_list.mojo | 40 ++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index f37890a35a..5ed799cc82 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -410,15 +410,16 @@ struct VariadicListMem[ # VariadicPack # ===----------------------------------------------------------------------===# +alias _AnyTypeMetaType = __mlir_type[`!lit.anytrait<`, AnyType, `>`] + -# TODO: We need to genericize VariadicPack over the kinds of types it holds, -# instead of erasing them to AnyType. This would allow packs of values known to -# be Stringable for example. @register_passable struct VariadicPack[ elt_is_mutable: __mlir_type.i1, lifetime: AnyLifetime[elt_is_mutable].type, - *element_types: AnyType, + element_trait: _AnyTypeMetaType, + *element_types: element_trait, + # TODO: Add address_space when Reference supports it. ](Sized): """A utility class to access variadic pack arguments and provide an API for doing things with them. @@ -427,12 +428,13 @@ struct VariadicPack[ elt_is_mutable: True if the elements of the list are mutable for an inout or owned argument pack. lifetime: The reference lifetime of the underlying elements. + element_trait: The trait that each element of the pack conforms to. element_types: The list of types held by the argument pack. """ alias _mlir_pack_type = __mlir_type[ `!lit.ref.pack<:variadic<`, - AnyType, + element_trait, `> `, element_types, `, `, @@ -502,7 +504,15 @@ struct VariadicPack[ fn get_element[ index: Int ](self) -> Reference[ - element_types[index.value], Self.elt_is_mutable, Self.lifetime + # FIXME: Shouldn't need a rebind here. + __mlir_attr[ + `#kgen.param.expr: `, + AnyType, + ], + Self.elt_is_mutable, + Self.lifetime, ]: """Return a reference to an element of the pack. @@ -514,8 +524,22 @@ struct VariadicPack[ mutability of the pack argument convention. """ - return __mlir_op.`lit.ref.pack.get`[index = index.value](self._value) - + return rebind[ + Reference[ + # FIXME: Shouldn't need a rebind here. + __mlir_attr[ + `#kgen.param.expr: `, + AnyType, + ], + Self.elt_is_mutable, + Self.lifetime, + ] + ](__mlir_op.`lit.ref.pack.get`[index = index.value](self._value)) + + # FIXME!: the T in the function should be element_trait bound not AnyType + # bound. @always_inline fn each[func: fn[T: AnyType] (T) -> None](self): """Apply a function to each element of the pack in order. This applies From d9282278315c0c7c2f31270af97e7bf80c46412c Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Mon, 1 Apr 2024 12:36:36 -0700 Subject: [PATCH 15/55] [mojo-lang] Finally add support for AnyTrait param conversion. (#36362) This teaches `canConvertWithRebind` that it is safe to rebind an parametric expression of type `AnyTrait[SomeTrait]` to `SomeTrait` which gets us in the parametric types game, and eliminates some grunge from `VariadicPack` implementation. modular-orig-commit: c34dc6183d5853c6960e92ff538256aa3e511f29 --- stdlib/src/builtin/builtin_list.mojo | 33 +++++++++++----------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 5ed799cc82..50ab6a44f8 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -504,13 +504,7 @@ struct VariadicPack[ fn get_element[ index: Int ](self) -> Reference[ - # FIXME: Shouldn't need a rebind here. - __mlir_attr[ - `#kgen.param.expr: `, - AnyType, - ], + element_types[index.value], Self.elt_is_mutable, Self.lifetime, ]: @@ -523,20 +517,19 @@ struct VariadicPack[ A reference to the element. The Reference's mutability follows the mutability of the pack argument convention. """ + var ref_elt = __mlir_op.`lit.ref.pack.get`[index = index.value]( + self._value + ) - return rebind[ - Reference[ - # FIXME: Shouldn't need a rebind here. - __mlir_attr[ - `#kgen.param.expr: `, - AnyType, - ], - Self.elt_is_mutable, - Self.lifetime, - ] - ](__mlir_op.`lit.ref.pack.get`[index = index.value](self._value)) + # Rebind the !lit.ref to agree on the element type. This is needed + # because we're getting a low level rebind to AnyType when the + # element_types[index] expression is erased to AnyType for Reference. + alias result_ref = Reference[ + element_types[index.value], + Self.elt_is_mutable, + Self.lifetime, + ] + return rebind[result_ref.mlir_ref_type](ref_elt) # FIXME!: the T in the function should be element_trait bound not AnyType # bound. From cfd479fb5e45f1cfaf824b0825adaf134b38d8a3 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Mon, 1 Apr 2024 17:15:23 -0600 Subject: [PATCH 16/55] [stdlib] Use variadic initializer for `List` in tests (#36410) Update the tests to make use of the variadic initializer of `List`. Upstream commit: [Internal Link] modular-orig-commit: eb3d9197014a73fef737574578429b68bed10808 --- stdlib/test/collections/test_list.mojo | 5 +---- stdlib/test/utils/issue_13632.mojo | 7 +------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/stdlib/test/collections/test_list.mojo b/stdlib/test/collections/test_list.mojo index 1b11735dbf..c634af2570 100644 --- a/stdlib/test/collections/test_list.mojo +++ b/stdlib/test/collections/test_list.mojo @@ -130,10 +130,7 @@ def test_list_reverse(): # Test reversing the list ["one", "two", "three"] # - vec2 = List[String]() - vec2.append("one") - vec2.append("two") - vec2.append("three") + vec2 = List[String]("one", "two", "three") assert_equal(len(vec2), 3) assert_equal(vec2[0], "one") diff --git a/stdlib/test/utils/issue_13632.mojo b/stdlib/test/utils/issue_13632.mojo index acf4117543..5299211e89 100644 --- a/stdlib/test/utils/issue_13632.mojo +++ b/stdlib/test/utils/issue_13632.mojo @@ -23,12 +23,7 @@ fn sum_items(data: List[Int8]) -> Int: fn make_abcd_vector() -> List[Int8]: - var v = List[Int8]() - v.append(97) - v.append(98) - v.append(99) - v.append(100) - return v + return List[Int8](97, 98, 99, 100) fn main(): From 3083a6cc3c40451c0786d1e5bbc64cf95398f81b Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Mon, 1 Apr 2024 16:48:00 -0700 Subject: [PATCH 17/55] [KGEN] Delete `POC::VariadicSize` (#36239) This was introduced as part of the previous attempt at managing autotuning (parameter-based variadic construction to feed into `kgen.param.fork`). This had one use in the standard library, but it is redundant with `pop.variadic.size`, so it's been replaced. modular-orig-commit: 2d1553790e03d2cc2ee5e24fe840b71728db155f --- stdlib/src/builtin/builtin_list.mojo | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 50ab6a44f8..850dd06a4c 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -487,9 +487,15 @@ struct VariadicPack[ Returns: The number of elements in the variadic pack. """ - return __mlir_attr[ - `#kgen.param.expr : index` - ] + + @parameter + fn variadic_size( + x: __mlir_type[`!kgen.variadic<`, AnyType, `>`] + ) -> Int: + return __mlir_op.`pop.variadic.size`(x) + + alias result = variadic_size(element_types) + return result @always_inline fn __len__(self) -> Int: From 6a8fb55309ed0a3ec024c54892173ef34319503c Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Mon, 1 Apr 2024 17:30:43 -0700 Subject: [PATCH 18/55] [mojo-stdlib] Hotfix merge conflict in main This fixes a merge conflict due to the change from VariadicPack working with general element types. modular-orig-commit: f62bb0a10e06107271ebf4dfa9e2db968bee4333 --- stdlib/src/builtin/builtin_list.mojo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 850dd06a4c..cc0bf0fc5a 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -490,7 +490,7 @@ struct VariadicPack[ @parameter fn variadic_size( - x: __mlir_type[`!kgen.variadic<`, AnyType, `>`] + x: __mlir_type[`!kgen.variadic<`, element_trait, `>`] ) -> Int: return __mlir_op.`pop.variadic.size`(x) From 17b1c47027c740399d9581ecebf1337b250609d8 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Tue, 2 Apr 2024 09:18:40 -0400 Subject: [PATCH 19/55] [mojo-stdlib] Make `PythonObject` conform to `KeyElement` (#36354) To do this, some dunder methods needed to be changed so that they don't raise (but abort instead) until `raises`-ness of functions can be parametric. This patch also adds a bit of glue to allow `Dict` to be used conveniently instead of a Python dictionary. modular-orig-commit: 612718161d13c7a074e0f1f98962788b8b0828da --- stdlib/src/python/_cpython.mojo | 5 +++ stdlib/src/python/object.mojo | 45 +++++++++++++++++++--- stdlib/test/python/test_python_object.mojo | 17 ++++++-- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/stdlib/src/python/_cpython.mojo b/stdlib/src/python/_cpython.mojo index 35ad5e6a9b..a3d5425025 100644 --- a/stdlib/src/python/_cpython.mojo +++ b/stdlib/src/python/_cpython.mojo @@ -453,6 +453,11 @@ struct CPython: ) ) + fn PyObject_Hash(inout self, obj: PyObjectPtr) -> Int: + return int( + self.lib.get_function[fn (PyObjectPtr) -> Int]("PyObject_Hash")(obj) + ) + fn PyTuple_New(inout self, count: Int) -> PyObjectPtr: var r = self.lib.get_function[fn (Int) -> PyObjectPtr]( StringRef("PyTuple_New") diff --git a/stdlib/src/python/object.mojo b/stdlib/src/python/object.mojo index 36461bcb95..4a1082a2cb 100644 --- a/stdlib/src/python/object.mojo +++ b/stdlib/src/python/object.mojo @@ -101,7 +101,7 @@ struct _PyIter(Sized): @register_passable struct PythonObject( - Intable, Stringable, SizedRaising, Boolable, CollectionElement + Intable, Stringable, SizedRaising, Boolable, CollectionElement, KeyElement ): """A Python object.""" @@ -295,6 +295,19 @@ struct PythonObject( unroll[fill, len(types)]() + fn __init__(inout self, value: Dict[Self, Self]): + """Initialize the object from a dictionary of PythonObjects. + + Args: + value: The dictionary value. + """ + var cpython = _get_global_python_itf().cpython() + self.py_object = cpython.PyDict_New() + for entry in value.items(): + var result = cpython.PyDict_SetItem( + self.py_object, entry[].key.py_object, entry[].value.py_object + ) + fn __copyinit__(inout self, existing: Self): """Copy the object. @@ -413,6 +426,18 @@ struct PythonObject( raise Error("object has no len()") return result + fn __hash__(self) -> Int: + """Returns the length of the object. + + Returns: + The length of the object. + """ + var cpython = _get_global_python_itf().cpython() + var result = cpython.PyObject_Length(self.py_object) + # TODO: make this function raise when we can raise parametrically. + debug_assert(result != -1, "object is not hashable") + return result + fn __getitem__(self, *args: PythonObject) raises -> PythonObject: """Return the value for the given key or keys. @@ -953,7 +978,7 @@ struct PythonObject( """ return self._call_single_arg_method("__ge__", rhs) - fn __eq__(self, rhs: PythonObject) raises -> PythonObject: + fn __eq__(self, rhs: PythonObject) -> Bool: """Equality comparator. This compares the elements of strings and lists. Args: @@ -962,9 +987,14 @@ struct PythonObject( Returns: True if the objects are equal. """ - return self._call_single_arg_method("__eq__", rhs) + # TODO: make this function raise when we can raise parametrically. + try: + return self._call_single_arg_method("__eq__", rhs).__bool__() + except e: + debug_assert(False, "object doesn't implement __eq__") + return False - fn __ne__(self, rhs: PythonObject) raises -> PythonObject: + fn __ne__(self, rhs: PythonObject) -> Bool: """Inequality comparator. This compares the elements of strings and lists. @@ -974,7 +1004,12 @@ struct PythonObject( Returns: True if the objects are not equal. """ - return self._call_single_arg_method("__ne__", rhs) + # TODO: make this function raise when we can raise parametrically. + try: + return self._call_single_arg_method("__ne__", rhs).__bool__() + except e: + debug_assert(False, "object doesn't implement __eq__") + return False fn __pos__(self) raises -> PythonObject: """Positive. diff --git a/stdlib/test/python/test_python_object.mojo b/stdlib/test/python/test_python_object.mojo index 16b9025e0d..e2243f1ab2 100644 --- a/stdlib/test/python/test_python_object.mojo +++ b/stdlib/test/python/test_python_object.mojo @@ -430,14 +430,14 @@ def test_is(): # CHECK-LABEL: test_iter -fn test_iter() raises -> None: +fn test_iter() raises: print("=== test_iter ===") - var list: PythonObject = ["apple", "orange", "banana"] + var list_obj: PythonObject = ["apple", "orange", "banana"] # CHECK: I like to eat apple # CHECK: I like to eat orange # CHECK: I like to eat banana - for fruit in list: + for fruit in list_obj: print("I like to eat", fruit) var list2: PythonObject = [] @@ -453,6 +453,16 @@ fn test_iter() raises -> None: assert_false(True) +fn test_dict() raises: + var d = Dict[PythonObject, PythonObject]() + d["food"] = 123 + d["fries"] = "yes" + + var dd = PythonObject(d) + # CHECK: {'food': 123, 'fries': 'yes'} + print(dd) + + def main(): # initializing Python instance calls init_python var python = Python() @@ -463,3 +473,4 @@ def main(): test_len() test_is() test_iter() + test_dict() From fa92b988b5118a4ac2ffc207620e9a3305064212 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Tue, 2 Apr 2024 08:09:00 -0600 Subject: [PATCH 20/55] [stdlib] Add `String.removeprefix` and `String.removesuffix` (#36411) The two methods `removeprefix` and `removesuffix` are present in python. See the docs: * https://docs.python.org/3/library/stdtypes.html#str.removeprefix * https://docs.python.org/3/library/stdtypes.html#str.removesuffix Both were introduced in python 3.9. The documentation was added, the arguments were made positional-only like in CPython. Unit tests were added in the corresponding module. The docstrings were taken from the python documentation. Note: - Had minor formatting fixes to please the internal doc-string validation since that is not yet enabled on the public CI. Upstream commit: [Internal Link] modular-orig-commit: 16a471fe050c17a236f5086adb092622aa7ac0ff --- stdlib/src/builtin/string.mojo | 42 ++++++++++++++++++++++++++++ stdlib/test/builtin/test_string.mojo | 16 +++++++++++ 2 files changed, 58 insertions(+) diff --git a/stdlib/src/builtin/string.mojo b/stdlib/src/builtin/string.mojo index ad35249844..280c0137dd 100644 --- a/stdlib/src/builtin/string.mojo +++ b/stdlib/src/builtin/string.mojo @@ -999,6 +999,48 @@ struct String(Sized, Stringable, IntableRaising, KeyElement, Boolable): return self._endswith_impl(suffix, start) return self[start:end]._endswith_impl(suffix) + fn removeprefix(self, prefix: String, /) -> String: + """If the string starts with the prefix string, return `string[len(prefix):]`. + Otherwise, return a copy of the original string. + + ```mojo + print(String('TestHook').removeprefix('Test')) + # 'Hook' + print(String('BaseTestCase').removeprefix('Test')) + # 'BaseTestCase' + ``` + + Args: + prefix: The prefix to remove from the string. + + Returns: + A new string with the prefix removed if it was present. + """ + if self.startswith(prefix): + return self[len(prefix) :] + return self + + fn removesuffix(self, suffix: String, /) -> String: + """If the string ends with the suffix string, return `string[:-len(suffix)]`. + Otherwise, return a copy of the original string. + + ```mojo + print(String('TestHook').removesuffix('Hook')) + # 'Test' + print(String('BaseTestCase').removesuffix('Test')) + # 'BaseTestCase' + ``` + + Args: + suffix: The suffix to remove from the string. + + Returns: + A new string with the suffix removed if it was present. + """ + if self.endswith(suffix): + return self[: -len(suffix)] + return self + @always_inline fn _endswith_impl(self, suffix: String, start: Int = 0) -> Bool: return self.rfind(suffix, start) + len(suffix) == len(self) diff --git a/stdlib/test/builtin/test_string.mojo b/stdlib/test/builtin/test_string.mojo index 65297ce6ad..2cbfc5db2a 100644 --- a/stdlib/test/builtin/test_string.mojo +++ b/stdlib/test/builtin/test_string.mojo @@ -545,6 +545,20 @@ fn test_endswith() raises: assert_false(str.endswith("llo", 2, 3)) +def test_removeprefix(): + assert_equal(String("hello world").removeprefix("hello"), " world") + assert_equal(String("hello world").removeprefix("world"), "hello world") + assert_equal(String("hello world").removeprefix("hello world"), "") + assert_equal(String("hello world").removeprefix("llo wor"), "hello world") + + +def test_removesuffix(): + assert_equal(String("hello world").removesuffix("world"), "hello ") + assert_equal(String("hello world").removesuffix("hello"), "hello world") + assert_equal(String("hello world").removesuffix("hello world"), "") + assert_equal(String("hello world").removesuffix("llo wor"), "hello world") + + def test_intable(): assert_equal(int(String("123")), 123) @@ -590,5 +604,7 @@ def main(): test_hash() test_startswith() test_endswith() + test_removeprefix() + test_removesuffix() test_intable() test_string_mul() From 28eca82f8e4d182652ea7e12be17929fd067acd8 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Tue, 2 Apr 2024 10:25:19 -0400 Subject: [PATCH 21/55] [mojo-stdlib] Teach `PythonObject` about `__setitem__` (#36355) This enables users to conveniently set elements of Python lists and dicts, among other things. modular-orig-commit: 815d912bf5ff0e726abdfc0868d374eee3972ab2 --- stdlib/src/python/object.mojo | 26 ++++++++++++++++++++++ stdlib/test/python/test_python_object.mojo | 18 ++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/stdlib/src/python/object.mojo b/stdlib/src/python/object.mojo index 4a1082a2cb..a16cbca4c5 100644 --- a/stdlib/src/python/object.mojo +++ b/stdlib/src/python/object.mojo @@ -466,6 +466,32 @@ struct PythonObject( Python.throw_python_exception_if_error_state(cpython) return PythonObject(result) + fn __setitem__(inout self, *args: PythonObject) raises: + """Set the value with the given key or keys. + + Args: + args: The key or keys to set on this object, followed by the value. + """ + var size = len(args) + debug_assert(size > 0, "must provide at least a value to __setitem__") + + var cpython = _get_global_python_itf().cpython() + var tuple_obj = cpython.PyTuple_New(size) + for i in range(size): + var arg_value = args[i].py_object + cpython.Py_IncRef(arg_value) + var result = cpython.PyTuple_SetItem(tuple_obj, i, arg_value) + if result != 0: + raise Error("internal error: PyTuple_SetItem failed") + + var callable_obj = cpython.PyObject_GetAttrString( + self.py_object, "__setitem__" + ) + var result = cpython.PyObject_CallObject(callable_obj, tuple_obj) + cpython.Py_DecRef(callable_obj) + cpython.Py_DecRef(tuple_obj) + Python.throw_python_exception_if_error_state(cpython) + fn _call_zero_arg_method( self, method_name: StringRef ) raises -> PythonObject: diff --git a/stdlib/test/python/test_python_object.mojo b/stdlib/test/python/test_python_object.mojo index e2243f1ab2..b88f001e1b 100644 --- a/stdlib/test/python/test_python_object.mojo +++ b/stdlib/test/python/test_python_object.mojo @@ -453,15 +453,30 @@ fn test_iter() raises: assert_false(True) +fn test_setitem() raises: + var ll = PythonObject([1, 2, 3, "food"]) + # CHECK: [1, 2, 3, 'food'] + print(ll) + ll[1] = "nomnomnom" + # CHECK: [1, 'nomnomnom', 3, 'food'] + print(ll) + + fn test_dict() raises: var d = Dict[PythonObject, PythonObject]() - d["food"] = 123 + d["food"] = "remove this" d["fries"] = "yes" + d["food"] = 123 # intentionally replace to ensure keys stay in order var dd = PythonObject(d) # CHECK: {'food': 123, 'fries': 'yes'} print(dd) + dd["food"] = "salad" + dd[42] = Python.evaluate("[4, 2]") + # CHECK: {'food': 'salad', 'fries': 'yes', 42: [4, 2]} + print(dd) + def main(): # initializing Python instance calls init_python @@ -473,4 +488,5 @@ def main(): test_len() test_is() test_iter() + test_setitem() test_dict() From 65268c8a1235b515475f5a74bcf0f27b0db7c6ec Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 2 Apr 2024 11:47:55 -0600 Subject: [PATCH 22/55] [mojo-lsp] Fix hover snippet for functions with functional args/params/result (#36391) The LSP was using `->` as a heuristic for the separation between the arguments and results of a function signature, which isn't correct in the presence of functional types. This PR fixes the rendering to properly track the offset of a return, to ensure the hover is always right. Closes [Internal Link] Closes [Internal Link] modular-orig-commit: ed82e2e607cda9c8d7a018d0d545a453443962c3 --- docs/changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index b6d02e059d..51695bb718 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -51,3 +51,8 @@ and tools. Please add any significant user-visible changes here. - [#1987](https://github.com/modularml/mojo/issues/1987) Defining `main` in a Mojo package is an error, for now. This is not intended to work yet, erroring for now will help to prevent accidental undefined behavior. + +- [#1215](https://github.com/modularml/mojo/issues/1215) and + [#1949](https://github.com/modularml/mojo/issues/1949) The Mojo LSP server no + longer cuts off hover previews for functions with functional arguments, + parameters, or results. From aff80b9da4f1350f153da88bd2e13f4ab3d34403 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 2 Apr 2024 13:19:59 -0600 Subject: [PATCH 23/55] [mojo-tooling] Fix processing of argument conventions in lsp/docs (#36408) This PR fixes the detection and processing of argument conventions, allowing properly handling and display of inout/owned/borrowed arguments (also respecting the defaults for fn vs def). Closes [Internal Link] modular-orig-commit: 83af6de2ec5e2091b3309ef12e5add3eaba2bb34 --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 51695bb718..d427ac5ab2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -56,3 +56,6 @@ and tools. Please add any significant user-visible changes here. [#1949](https://github.com/modularml/mojo/issues/1949) The Mojo LSP server no longer cuts off hover previews for functions with functional arguments, parameters, or results. + +- [#1901](https://github.com/modularml/mojo/issues/1901) Fixed Mojo LSP and + documentation generation handling of inout arguments. From 2ce7ba3c422b4b3eba8d5036ff0b223e7d1efd30 Mon Sep 17 00:00:00 2001 From: Walter Erquinigo Date: Tue, 2 Apr 2024 18:38:07 -0400 Subject: [PATCH 24/55] [SDK][Debug] Enable JIT debugging in the mac sdk (#36220) Closes [Internal Link] This also enables the self test on mac, which passes in [Internal Link] modular-orig-commit: 05cb1a0f5b903d09186a2e48554237d03de0f17b --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index d427ac5ab2..5f5fcc1aeb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -59,3 +59,6 @@ and tools. Please add any significant user-visible changes here. - [#1901](https://github.com/modularml/mojo/issues/1901) Fixed Mojo LSP and documentation generation handling of inout arguments. + +- [#1924](https://github.com/modularml/mojo/issues/1924) JIT debugging on Mac + has been fixed. From c75fdbfdec2cb3095fd0a4a3f76bc0a6b0801067 Mon Sep 17 00:00:00 2001 From: akirchhoff-modular Date: Tue, 2 Apr 2024 17:38:37 -0700 Subject: [PATCH 25/55] [KGEN] Fix crash when parsing `0__` (#35899) There was an assumption that if a literal starts with `0` and is at least 3 characters long, it has to be because it started with `0b`, `0x`, etc., and hence was safe to strip off 2 characters. When you do this with `0__`, you end up with `_`, which after filtering underscores, is an empty string, which can't be parsed as an integer, causing an assertion error. Instead, the first two characters only if it's one of the prefixes we expect. Fixes modularml/mojo#1913. modular-orig-commit: 75cce9ad8716af3f950a5e36fa855a343f65ba16 --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 5f5fcc1aeb..fb95f5f982 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -60,5 +60,8 @@ and tools. Please add any significant user-visible changes here. - [#1901](https://github.com/modularml/mojo/issues/1901) Fixed Mojo LSP and documentation generation handling of inout arguments. +- [#1913](https://github.com/modularml/mojo/issues/1913) - `0__` no longer + crashes the Mojo parser. + - [#1924](https://github.com/modularml/mojo/issues/1924) JIT debugging on Mac has been fixed. From ae3837fd25716c4cca94e540e86a6553a89c41e4 Mon Sep 17 00:00:00 2001 From: abdul dakkak Date: Tue, 2 Apr 2024 17:54:46 -0700 Subject: [PATCH 26/55] [Stdlib] Add a load method to Atomic (#36495) This gives the current underlying value for the atomic. Closes #36470 modular-orig-commit: d5c0d002492b896d91600e8af6306301f1219a80 --- stdlib/src/os/atomic.mojo | 11 ++++++++++- stdlib/test/os/test_atomic.mojo | 7 +++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/stdlib/src/os/atomic.mojo b/stdlib/src/os/atomic.mojo index f75175ab27..ec5c0ab592 100644 --- a/stdlib/src/os/atomic.mojo +++ b/stdlib/src/os/atomic.mojo @@ -55,7 +55,16 @@ struct Atomic[type: DType]: Args: value: Initial value represented as `mlir.index` type. """ - self.__init__(Scalar[type](value)) + self.value = value + + @always_inline + fn load(inout self) -> Scalar[type]: + """Loads the current value from the atomic. + + Returns: + The current value of the atomic. + """ + return self.fetch_add(0) @staticmethod @always_inline diff --git a/stdlib/test/os/test_atomic.mojo b/stdlib/test/os/test_atomic.mojo index c1546d0da6..563231e662 100644 --- a/stdlib/test/os/test_atomic.mojo +++ b/stdlib/test/os/test_atomic.mojo @@ -21,6 +21,9 @@ fn test_atomic(): var atom: Atomic[DType.index] = 3 + # CHECK: 3 + print(atom.load()) + # CHECK: 3 print(atom.value) @@ -52,7 +55,7 @@ fn test_atomic(): # CHECK-LABEL: test_atomic_floating_point -fn test_atomic_floating_poInt__(): +fn test_atomic_floating_point(): print("== test_atomic_floating_point") var atom: Atomic[DType.float32] = Float32(3.0) @@ -89,4 +92,4 @@ fn test_atomic_floating_poInt__(): fn main(): test_atomic() - test_atomic_floating_poInt__() + test_atomic_floating_point() From b023521b0e5bb571dc0028435e9bbabfd96d6a30 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Tue, 2 Apr 2024 23:21:51 -0700 Subject: [PATCH 27/55] [mojo-lang] Fix meta-trait type parameter rebinding logic. (#36584) This enhances type checking of type parameter expressions of AnyTrait type to look through downcasts of the type itself and allows downcast from a value of `AnyTrait[SomeTrait]` to `SomeTrait`. This is enough to enable us to type check the fancy dependently typed `each` method on `VariadicPack` which makes VariadicPack useful for non-AnyType trait requirements as shown in the integration test!!! This bumps up against a bug in the KGEN ParameterVerifier. I believe the fix is simple, but am not 100% confident about it, so I temporarily disabled it and filed #36583 to discuss this. modular-orig-commit: 816c623595bce4a995afc795bf5fd551c0c83900 --- stdlib/src/builtin/builtin_list.mojo | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index cc0bf0fc5a..648907336e 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -537,10 +537,8 @@ struct VariadicPack[ ] return rebind[result_ref.mlir_ref_type](ref_elt) - # FIXME!: the T in the function should be element_trait bound not AnyType - # bound. @always_inline - fn each[func: fn[T: AnyType] (T) -> None](self): + fn each[func: fn[T: element_trait] (T) -> None](self): """Apply a function to each element of the pack in order. This applies the specified function (which must be parametric on the element type) to each element of the pack, from the first element to the last, passing @@ -552,6 +550,6 @@ struct VariadicPack[ @parameter fn unrolled[i: Int](): - func(self.get_element[i]()[]) + func[element_types[i.value]](self.get_element[i]()[]) unroll[unrolled, Self.__len__()]() From 82095db521b2d3acae8e7c5b6752c8ca90c5476e Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 09:07:10 -0400 Subject: [PATCH 28/55] [mojo-stdlib] Add back `Python.dict()` (#36499) This was previously removed, but is actually used by our manual and also is generally useful. Since `Dict` can now be used by the interop, implementing it is trivial. modular-orig-commit: cd0cc86e047f4e49c57969942132f9dda03f2e4f --- stdlib/src/python/python.mojo | 9 +++++++++ stdlib/test/python/test_python_object.mojo | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/stdlib/src/python/python.mojo b/stdlib/src/python/python.mojo index d8af8ba9ea..86063e4ecc 100644 --- a/stdlib/src/python/python.mojo +++ b/stdlib/src/python/python.mojo @@ -170,6 +170,15 @@ struct Python: Python.throw_python_exception_if_error_state(cpython) return PythonObject(module_maybe) + @staticmethod + fn dict() -> PythonObject: + """Construct an empty Python dictionary. + + Returns: + The constructed empty Python dictionary. + """ + return PythonObject(Dict[PythonObject, PythonObject]()) + fn __str__(inout self, str_obj: PythonObject) -> StringRef: """Return a string representing the given Python object. diff --git a/stdlib/test/python/test_python_object.mojo b/stdlib/test/python/test_python_object.mojo index b88f001e1b..f430adbd97 100644 --- a/stdlib/test/python/test_python_object.mojo +++ b/stdlib/test/python/test_python_object.mojo @@ -477,6 +477,11 @@ fn test_dict() raises: # CHECK: {'food': 'salad', 'fries': 'yes', 42: [4, 2]} print(dd) + # Also test that Python.dict() creates the right object. + var empty = Python.dict() + # CHECK: empty: {} + print("empty:", empty) + def main(): # initializing Python instance calls init_python From b7ca25002879445281d9012bc97265fc9a4b9e79 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 09:12:17 -0400 Subject: [PATCH 29/55] [Mojo][Docs] Update changelog with fix to Python dict interop (#36446) modular-orig-commit: ec6fa3cea82422f85f5be7f08f0d93f3222ea1d1 --- docs/changelog.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index fb95f5f982..f6fb4d3ae3 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,6 +31,19 @@ and tools. Please add any significant user-visible changes here. shorter alias is equivalent to writing `--debug-level full`. This option is also available in the `mojo debug` command, but is already the default. +- `PythonObject` now conforms to the `KeyElement` trait, meaning that it can be + used as key type for `Dict`. This allows on to easily build and interact with + Python dictionaries in mojo: + + ```mojo + def main(): + d = PythonObject(Dict[PythonObject, PythonObject]()) + d["foo"] = 12 + d[7] = "bar" + d["foo"] = [1, 2, "something else"] + print(d) # prints `{'foo': [1, 2, 'something else'], 7: 'bar'}` + ``` + ### πŸ¦‹ Changed - The behavior of `mojo build` when invoked without an output `-o` argument has From 25c92b323bc90c2249279c62ae80d501fd79b77c Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 10:00:20 -0400 Subject: [PATCH 30/55] [mojo-stdlib] Add `Python.list()` method (#36504) To make it easier to create an empty python list. modular-orig-commit: ab43b2587bb32db7195a8f45cafb71279fb7e35a --- stdlib/src/python/python.mojo | 9 +++++++++ stdlib/test/python/test_python_object.mojo | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/stdlib/src/python/python.mojo b/stdlib/src/python/python.mojo index 86063e4ecc..5ceb5093cb 100644 --- a/stdlib/src/python/python.mojo +++ b/stdlib/src/python/python.mojo @@ -179,6 +179,15 @@ struct Python: """ return PythonObject(Dict[PythonObject, PythonObject]()) + @staticmethod + fn list() -> PythonObject: + """Construct an empty Python list. + + Returns: + The constructed empty Python list. + """ + return PythonObject([]) + fn __str__(inout self, str_obj: PythonObject) -> StringRef: """Return a string representing the given Python object. diff --git a/stdlib/test/python/test_python_object.mojo b/stdlib/test/python/test_python_object.mojo index f430adbd97..d291319cda 100644 --- a/stdlib/test/python/test_python_object.mojo +++ b/stdlib/test/python/test_python_object.mojo @@ -394,7 +394,7 @@ fn test_string_conversions() -> None: # CHECK-LABEL: test_len def test_len(): print("=== test_len ===") - var empty_list = Python.evaluate("[]") + var empty_list = Python.list() # CHECK: 0 print(len(empty_list)) From 8724ea5b358f5c1139cb71fc0987f2bc85fed142 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 10:46:42 -0400 Subject: [PATCH 31/55] [mojo-stdlib] Simplify `Python.none()` (#36505) `PythonObject` already implements the logic to create a Python `None`, so there this method can just simply delegate. The patch also adds a test case for this. modular-orig-commit: 629355fcfd56b1dba249550c894193f69f71891c --- stdlib/src/python/python.mojo | 5 +---- stdlib/test/python/test_python_object.mojo | 8 ++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/stdlib/src/python/python.mojo b/stdlib/src/python/python.mojo index 5ceb5093cb..546acdce4d 100644 --- a/stdlib/src/python/python.mojo +++ b/stdlib/src/python/python.mojo @@ -249,7 +249,4 @@ struct Python: Returns: `PythonObject` representing `None`. """ - var cpython = _get_global_python_itf().cpython() - var none = cpython.Py_None() - cpython.Py_IncRef(none) - return PythonObject(none) + return PythonObject(None) diff --git a/stdlib/test/python/test_python_object.mojo b/stdlib/test/python/test_python_object.mojo index d291319cda..0b200f9388 100644 --- a/stdlib/test/python/test_python_object.mojo +++ b/stdlib/test/python/test_python_object.mojo @@ -483,6 +483,13 @@ fn test_dict() raises: print("empty:", empty) +fn test_none() raises: + var n = Python.none() + # CHECK: None from Python: None + print("None from Python: ", n) + assert_true(n is None) + + def main(): # initializing Python instance calls init_python var python = Python() @@ -495,3 +502,4 @@ def main(): test_iter() test_setitem() test_dict() + test_none() From 969a8e70728149f8d0442fd2593e634fada7eb7e Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 10:05:42 -0600 Subject: [PATCH 32/55] [stdlib] Fix base64 encode for non-ASCII chars (#36598) This change fixes following bug: [Internal Link] Upstream commit: [Internal Link] modular-orig-commit: ab9ccb453b1c928fde1d8ac69c382d2dfa925a6b --- stdlib/src/base64/base64.mojo | 2 +- stdlib/test/base64/test_base64.mojo | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/stdlib/src/base64/base64.mojo b/stdlib/src/base64/base64.mojo index 9fbd0c933e..fe02bbb26b 100644 --- a/stdlib/src/base64/base64.mojo +++ b/stdlib/src/base64/base64.mojo @@ -47,7 +47,7 @@ fn b64encode(str: String) -> String: @parameter @always_inline fn s(idx: Int) -> Int: - return int(str._buffer[idx]) + return int(str._as_ptr().bitcast[DType.uint8]()[idx]) # This algorithm is based on https://arxiv.org/abs/1704.00605 var end = length - (length % 3) diff --git a/stdlib/test/base64/test_base64.mojo b/stdlib/test/base64/test_base64.mojo index 310a606892..934a87e57c 100644 --- a/stdlib/test/base64/test_base64.mojo +++ b/stdlib/test/base64/test_base64.mojo @@ -28,6 +28,9 @@ fn test_b64encode(): # CHECK: SGVsbG8gTW9qbyEhIQ== print(b64encode("Hello Mojo!!!")) + # CHECK: SGVsbG8g8J+UpSEhIQ== + print(b64encode("Hello πŸ”₯!!!")) + # CHECK: dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw== print(b64encode("the quick brown fox jumps over the lazy dog")) From a91194e4a7b9ce2e0fa0e242b272e27a349fdfb6 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 12:14:43 -0400 Subject: [PATCH 33/55] [mojo-stdlib] Introduce `OwnedKwargsDict` (#36468) This new dictionary type is a wrapper over `Dict`, specializing on `String` keys. In a subsequent patch, this type will be used by the parser to pass variadic keyword arguments that are declared with `owned` ownership semantics. modular-orig-commit: ed73c202fa15d38f543f1baee525635b6aa989b4 --- stdlib/src/collections/dict.mojo | 210 ++++++++++++++++++++++++- stdlib/test/collections/test_dict.mojo | 62 +++++++- 2 files changed, 267 insertions(+), 5 deletions(-) diff --git a/stdlib/src/collections/dict.mojo b/stdlib/src/collections/dict.mojo index 426745c31d..bdde7534c5 100644 --- a/stdlib/src/collections/dict.mojo +++ b/stdlib/src/collections/dict.mojo @@ -493,7 +493,7 @@ struct Dict[K: KeyElement, V: CollectionElement](Sized, CollectionElement): self._insert(key, value) fn __contains__(self, key: K) -> Bool: - """Check if a given value is in the dictionary or not. + """Check if a given key is in the dictionary or not. Args: key: The key to check. @@ -749,3 +749,211 @@ struct Dict[K: KeyElement, V: CollectionElement](Sized, CollectionElement): self._entries[right] = None self._n_entries = self.size + + +struct OwnedKwargsDict[V: CollectionElement](Sized, CollectionElement): + """Container used to pass owned variadic keyword arguments to functions. + + This type mimics the interface of a dictionary with `String` keys, and + should be usable more-or-less like a dictionary. Notably, however, this type + should not be instantiated directly by users. + + Parameters: + V: The value type of the dictionary. Currently must be CollectionElement. + """ + + alias key_type = String + + var _dict: Dict[Self.key_type, V] + + fn __init__(inout self): + """Initialize an empty keyword dictionary.""" + self._dict = Dict[Self.key_type, V]() + + fn __copyinit__(inout self, existing: Self): + """Copy an existing keyword dictionary. + + Args: + existing: The existing keyword dictionary. + """ + self._dict = existing._dict + + fn __moveinit__(inout self, owned existing: Self): + """Move data of an existing keyword dictionary into a new one. + + Args: + existing: The existing keyword dictionary. + """ + self._dict = existing._dict^ + + @always_inline("nodebug") + fn __getitem__(self, key: Self.key_type) raises -> V: + """Retrieve a value out of the keyword dictionary. + + Args: + key: The key to retrieve. + + Returns: + The value associated with the key, if it's present. + + Raises: + "KeyError" if the key isn't present. + """ + return self._dict[key] + + @always_inline("nodebug") + fn __setitem__(inout self, key: Self.key_type, value: V): + """Set a value in the keyword dictionary by key. + + Args: + key: The key to associate with the specified value. + value: The data to store in the dictionary. + """ + self._dict[key] = value + + @always_inline("nodebug") + fn __contains__(self, key: Self.key_type) -> Bool: + """Check if a given key is in the keyword dictionary or not. + + Args: + key: The key to check. + + Returns: + True if there key exists in the keyword dictionary, False + otherwise. + """ + return key in self._dict + + @always_inline("nodebug") + fn __len__(self) -> Int: + """The number of elements currenly stored in the keyword dictionary.""" + return len(self._dict) + + @always_inline("nodebug") + fn find(self, key: Self.key_type) -> Optional[V]: + """Find a value in the keyword dictionary by key. + + Args: + key: The key to search for in the dictionary. + + Returns: + An optional value containing a copy of the value if it was present, + otherwise an empty Optional. + """ + return self._dict.find(key) + + @always_inline("nodebug") + fn pop( + inout self, key: self.key_type, owned default: Optional[V] = None + ) raises -> V: + """Remove a value from the keyword dictionary by key. + + Args: + key: The key to remove from the dictionary. + default: Optionally provide a default value to return if the key + was not found instead of raising. + + Returns: + The value associated with the key, if it was in the dictionary. + If it wasn't, return the provided default value instead. + + Raises: + "KeyError" if the key was not present in the dictionary and no + default value was provided. + """ + return self._dict.pop(key, default^) + + fn __iter__[ + mutability: __mlir_type.`i1`, self_life: AnyLifetime[mutability].type + ]( + self: Reference[Self, mutability, self_life].mlir_ref_type, + ) -> _DictKeyIter[Self.key_type, V, mutability, self_life]: + """Iterate over the keyword dict's keys as immutable references. + + Parameters: + mutability: Whether the dict is mutable. + self_life: The dict's lifetime. + + Returns: + An iterator of immutable references to the dictionary keys. + """ + # TODO(#36448): Use this instead of the current workaround + # return self._dict.__iter__() + return _DictKeyIter( + _DictEntryIter[Self.key_type, V, mutability, self_life]( + 0, 0, Reference(self)[]._dict + ) + ) + + fn keys[ + mutability: __mlir_type.`i1`, self_life: AnyLifetime[mutability].type + ]( + self: Reference[Self, mutability, self_life].mlir_ref_type, + ) -> _DictKeyIter[Self.key_type, V, mutability, self_life]: + """Iterate over the keyword dict's keys as immutable references. + + Parameters: + mutability: Whether the dict is mutable. + self_life: The dict's lifetime. + + Returns: + An iterator of immutable references to the dictionary keys. + """ + # TODO(#36448): Use this instead of the current workaround + # return self._dict.keys() + return Self.__iter__(self) + + fn values[ + mutability: __mlir_type.`i1`, self_life: AnyLifetime[mutability].type + ]( + self: Reference[Self, mutability, self_life].mlir_ref_type, + ) -> _DictValueIter[Self.key_type, V, mutability, self_life]: + """Iterate over the keyword dict's values as references. + + Parameters: + mutability: Whether the dict is mutable. + self_life: The dict's lifetime. + + Returns: + An iterator of references to the dictionary values. + """ + # TODO(#36448): Use this instead of the current workaround + # return self._dict.values() + return _DictValueIter( + _DictEntryIter[Self.key_type, V, mutability, self_life]( + 0, 0, Reference(self)[]._dict + ) + ) + + fn items[ + mutability: __mlir_type.`i1`, self_life: AnyLifetime[mutability].type + ]( + self: Reference[Self, mutability, self_life].mlir_ref_type, + ) -> _DictEntryIter[Self.key_type, V, mutability, self_life]: + """Iterate over the keyword dictionary's entries as immutable references. + + These can't yet be unpacked like Python dict items, but you can + access the key and value as attributes ie. + + ```mojo + for e in dict.items(): + print(e[].key, e[].value) + ``` + + Parameters: + mutability: Whether the dict is mutable. + self_life: The dict's lifetime. + + Returns: + An iterator of immutable references to the dictionary entries. + """ + + # TODO(#36448): Use this instead of the current workaround + # return Reference(self)[]._dict.items() + return _DictEntryIter[Self.key_type, V, mutability, self_life]( + 0, 0, Reference(self)[]._dict + ) + + @always_inline("nodebug") + fn _insert(inout self, owned key: Self.key_type, owned value: V): + self._dict._insert(key^, value^) diff --git a/stdlib/test/collections/test_dict.mojo b/stdlib/test/collections/test_dict.mojo index ac4ccfb83f..f4afc36612 100644 --- a/stdlib/test/collections/test_dict.mojo +++ b/stdlib/test/collections/test_dict.mojo @@ -13,7 +13,7 @@ # RUN: %mojo -debug-level full %s from collections import Optional -from collections.dict import Dict, KeyElement +from collections.dict import Dict, KeyElement, OwnedKwargsDict from test_utils import CopyCounter from testing import * @@ -53,7 +53,7 @@ def test_compact(): for i in range(20): var key = "key" + str(i) dict[key] = i + 1 - dict.pop(key) + _ = dict.pop(key) assert_equal(0, len(dict)) @@ -182,7 +182,7 @@ def test_dict_copy_add_new_item(): def test_dict_copy_calls_copy_constructor(): var orig = Dict[String, CopyCounter]() - orig["a"] = CopyCounter()^ + orig["a"] = CopyCounter() # test values copied to new Dict var copy = Dict(orig) @@ -203,7 +203,7 @@ fn test[name: String, test_fn: fn () raises -> object]() raises: print("PASS") -def main(): +def test_dict(): test_dict_construction() test["test_basic", test_basic]() test["test_multiple_resizes", test_multiple_resizes]() @@ -223,3 +223,57 @@ def main(): "test_dict_copy_calls_copy_constructor", test_dict_copy_calls_copy_constructor, ]() + + +def test_taking_owned_kwargs_dict(owned kwargs: OwnedKwargsDict[Int]): + assert_equal(len(kwargs), 2) + + assert_true("fruit" in kwargs) + assert_equal(kwargs["fruit"], 8) + assert_equal(kwargs["fruit"], 8) + + assert_true("dessert" in kwargs) + assert_equal(kwargs["dessert"], 9) + assert_equal(kwargs["dessert"], 9) + + var keys = String("") + for key in kwargs.keys(): + keys += key[] + assert_equal(keys, "fruitdessert") + + var sum = 0 + for val in kwargs.values(): + sum += val[] + assert_equal(sum, 17) + + assert_false(kwargs.find("salad").__bool__()) + with assert_raises(contains="KeyError"): + _ = kwargs["salad"] + + kwargs["salad"] = 10 + assert_equal(kwargs["salad"], 10) + + assert_equal(kwargs.pop("fruit"), 8) + assert_equal(kwargs.pop("fruit", 2), 2) + with assert_raises(contains="KeyError"): + _ = kwargs.pop("fruit") + + keys = String("") + sum = 0 + for entry in kwargs.items(): + keys += entry[].key + sum += entry[].value + assert_equal(keys, "dessertsalad") + assert_equal(sum, 19) + + +def test_owned_kwargs_dict(): + var owned_kwargs = OwnedKwargsDict[Int]() + owned_kwargs._insert("fruit", 8) + owned_kwargs._insert("dessert", 9) + test_taking_owned_kwargs_dict(owned_kwargs^) + + +def main(): + test_dict() + test_owned_kwargs_dict() From f88491cae5f4f2bf065041e71109d6c757e467ce Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 10:43:15 -0600 Subject: [PATCH 34/55] [stdlib] Add `list.pop(index)` API (#36597) Add `list.pop(index)` API to the stdlib as requested by [Internal Link] This API should handle the same functionality as the Python `list.pop`. Unit tests includes: - popping both positive and negative index - making sure returned value is consistent - making sure list len is consistent Upstream commit: [Internal Link] modular-orig-commit: 2077b615fd49acdce1287ce587a21c8342f0da1d --- stdlib/src/collections/list.mojo | 25 ++++++++++++++++++++ stdlib/test/collections/test_list.mojo | 32 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/stdlib/src/collections/list.mojo b/stdlib/src/collections/list.mojo index be94ae7ac1..a56be9b7b6 100644 --- a/stdlib/src/collections/list.mojo +++ b/stdlib/src/collections/list.mojo @@ -239,6 +239,31 @@ struct List[T: CollectionElement](CollectionElement, Sized): self._realloc(self.capacity // 2) return ret_val^ + @always_inline + fn pop(inout self, i: Int = -1) -> T: + """Pops a value from the list at the given index. + + Args: + i: The index of the value to pop. + + Returns: + The popped value. + """ + debug_assert(-self.size <= i < self.size, "pop index out of range") + + var normalized_idx = i + if i < 0: + normalized_idx += len(self) + + var ret_val = (self.data + normalized_idx).take_value() + for j in range(normalized_idx + 1, self.size): + (self.data + j).move_into(self.data + j - 1) + self.size -= 1 + if self.size * 4 < self.capacity: + if self.capacity > 1: + self._realloc(self.capacity // 2) + return ret_val^ + @always_inline fn reserve(inout self, new_capacity: Int): """Reserves the requested capacity. diff --git a/stdlib/test/collections/test_list.mojo b/stdlib/test/collections/test_list.mojo index c634af2570..6e3de70d17 100644 --- a/stdlib/test/collections/test_list.mojo +++ b/stdlib/test/collections/test_list.mojo @@ -85,6 +85,37 @@ def test_list(): assert_equal(2, list.capacity) +def test_list_pop(): + var list = List[Int]() + # Test pop with index + for i in range(6): + list.append(i) + + # try poping from index 3 for 3 times + for i in range(3, 6): + assert_equal(i, list.pop(3)) + + # list should have 3 elements now + assert_equal(3, len(list)) + assert_equal(0, list[0]) + assert_equal(1, list[1]) + assert_equal(2, list[2]) + + # Test pop with negative index + for i in range(0, 2): + assert_equal(i, list.pop(-len(list))) + + # test default index as well + assert_equal(2, list.pop()) + list.append(2) + assert_equal(2, list.pop()) + + # list should be empty now + assert_equal(0, len(list)) + # capacity should be 1 according to shrink_to_fit behavior + assert_equal(1, list.capacity) + + def test_list_variadic_constructor(): var l = List[Int](2, 4, 6) assert_equal(3, len(l)) @@ -450,6 +481,7 @@ def test_list_span(): def main(): test_mojo_issue_698() test_list() + test_list_pop() test_list_variadic_constructor() test_list_reverse() test_list_reverse_move_count() From 2c17046f49c0ae11020ddef2539d9b22a8708fc1 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 12:20:30 -0600 Subject: [PATCH 35/55] [stdlib] Remove `List.pop_back()` (#36612) Now that `List` has a `pop(index)` API, remove `pop_back()` from `List`. `pop()` without any arguments has the same semantics. Fix up the call sites internally in the standard library. While here, adjust `pop(index)` to consistently use `len(self)`. Add a changelog entry for the new API in `List` and the removal of `pop_back()`. modular-orig-commit: 76c8b15ac5c2dd2da879de4fd1f438dad3572c7d --- docs/changelog.md | 7 ++++++ stdlib/src/builtin/string.mojo | 2 +- stdlib/src/collections/list.mojo | 16 +------------ stdlib/test/collections/test_list.mojo | 32 +++++++------------------- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index f6fb4d3ae3..253a01a55a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -44,6 +44,10 @@ and tools. Please add any significant user-visible changes here. print(d) # prints `{'foo': [1, 2, 'something else'], 7: 'bar'}` ``` +- `List` collection now has a `pop(index)` API for removing an element + at a particular index. By default, `List.pop()` removes the last element + in the list. + ### πŸ¦‹ Changed - The behavior of `mojo build` when invoked without an output `-o` argument has @@ -59,6 +63,9 @@ and tools. Please add any significant user-visible changes here. ### ❌ Removed +- `List.pop_back()` has been removed. Use `List.pop()` instead which defaults + to popping the last element in the list. + ### πŸ› οΈ Fixed - [#1987](https://github.com/modularml/mojo/issues/1987) Defining `main` diff --git a/stdlib/src/builtin/string.mojo b/stdlib/src/builtin/string.mojo index 280c0137dd..269f210349 100644 --- a/stdlib/src/builtin/string.mojo +++ b/stdlib/src/builtin/string.mojo @@ -681,7 +681,7 @@ struct String(Sized, Stringable, IntableRaising, KeyElement, Boolable): # TODO(lifetimes): Return a reference rather than a copy var copy = self._buffer - var last = copy.pop_back() + var last = copy.pop() debug_assert( last == 0, "expected last element of String buffer to be null terminator", diff --git a/stdlib/src/collections/list.mojo b/stdlib/src/collections/list.mojo index a56be9b7b6..21387a5cdd 100644 --- a/stdlib/src/collections/list.mojo +++ b/stdlib/src/collections/list.mojo @@ -225,20 +225,6 @@ struct List[T: CollectionElement](CollectionElement, Sized): # list. self.size = final_size - @always_inline - fn pop_back(inout self) -> T: - """Pops a value from the back of this list. - - Returns: - The popped value. - """ - var ret_val = (self.data + (self.size - 1)).take_value() - self.size -= 1 - if self.size * 4 < self.capacity: - if self.capacity > 1: - self._realloc(self.capacity // 2) - return ret_val^ - @always_inline fn pop(inout self, i: Int = -1) -> T: """Pops a value from the list at the given index. @@ -249,7 +235,7 @@ struct List[T: CollectionElement](CollectionElement, Sized): Returns: The popped value. """ - debug_assert(-self.size <= i < self.size, "pop index out of range") + debug_assert(-len(self) <= i < len(self), "pop index out of range") var normalized_idx = i if i < 0: diff --git a/stdlib/test/collections/test_list.mojo b/stdlib/test/collections/test_list.mojo index 6e3de70d17..de74a5286f 100644 --- a/stdlib/test/collections/test_list.mojo +++ b/stdlib/test/collections/test_list.mojo @@ -57,32 +57,15 @@ def test_list(): list[-1] = 7 assert_equal(7, list[-1]) - # pop_back shall return the last element - # and adjust the size - assert_equal(7, list.pop_back()) - assert_equal(4, len(list)) - - # Verify that capacity shrinks as the list goes smaller - while list.size > 1: - _ = list.pop_back() - - assert_equal(1, len(list)) - assert_equal( - 1, list.size - ) # pedantically ensure len and size refer to the same thing - assert_equal(4, list.capacity) - - # Verify that capacity doesn't become 0 when the list gets empty. - _ = list.pop_back() - assert_equal(0, len(list)) - - # FIXME: revisit that pop_back is actually doing shrink_to_fit behavior - # under the hood which will be surprising to users - assert_equal(2, list.capacity) +def test_list_clear(): + var list = List[Int](1, 2, 3) + assert_equal(len(list), 3) + assert_equal(list.capacity, 3) list.clear() - assert_equal(0, len(list)) - assert_equal(2, list.capacity) + + assert_equal(len(list), 0) + assert_equal(list.capacity, 3) def test_list_pop(): @@ -481,6 +464,7 @@ def test_list_span(): def main(): test_mojo_issue_698() test_list() + test_list_clear() test_list_pop() test_list_variadic_constructor() test_list_reverse() From 0c4289ccf7cfd084d208c73ec5e6012813bbafaa Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Wed, 3 Apr 2024 11:36:36 -0700 Subject: [PATCH 36/55] [mojo-lang] Teach parameter inference about type rebinds. (#36587) We get type rebinds when metatypes are downcast - look through these to infer from the original type before mapping. In the example, we are allowed to infer that the argument has type `element_types[i]` even though the Reference type erased it down to `AnyType`. modular-orig-commit: 8212deaf37c4b83a75195a86ae08bb2b931d4997 --- stdlib/src/builtin/builtin_list.mojo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 648907336e..9913d106d8 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -550,6 +550,6 @@ struct VariadicPack[ @parameter fn unrolled[i: Int](): - func[element_types[i.value]](self.get_element[i]()[]) + func(self.get_element[i]()[]) unroll[unrolled, Self.__len__()]() From 4487840dc4f8f9d4888a44461dfd2f139cadf309 Mon Sep 17 00:00:00 2001 From: Arthur Evans Date: Wed, 3 Apr 2024 12:25:36 -0700 Subject: [PATCH 37/55] [Docs] Fix changelog formatting. Fixes #36539 (#36634) Why an extra blank line inside a fenced code block causes our MD processor to lose its mind is another question, but it does, and this fixes the issue in this instance. modular-orig-commit: 785e243082cedb89a5e5d0719ca33205bef441c2 --- docs/changelog-released.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/changelog-released.md b/docs/changelog-released.md index 0b83b56536..7860d298d9 100644 --- a/docs/changelog-released.md +++ b/docs/changelog-released.md @@ -1161,7 +1161,6 @@ installation issues. Otherwise it is functionally identical to Mojo 24.1. fn __copyinit__(inout self: Self, existing: Self): self.vec = existing.vec - fn main(): var foo = Foo() print(len(foo.vec)) From 0a6b5a10ea77531e4ceb8f7eec844f7338e26588 Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Wed, 3 Apr 2024 18:18:10 -0400 Subject: [PATCH 38/55] [mojo-lang][mojo-stdlib] Use `OwnedKwargsDict` for emitting variadic keyword arguments (#36540) Instead of using a `Dict`, the parser will use a more specialized type for passing variadic keyword arguments into callees. The patch also simplifies some of the emission logic by moving `String` creation for the keys into ``OwnedKwargsDict`. modular-orig-commit: bc0bc6edced025903ee095fa5ca3bea248690e66 --- stdlib/src/collections/dict.mojo | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stdlib/src/collections/dict.mojo b/stdlib/src/collections/dict.mojo index bdde7534c5..40f81e563a 100644 --- a/stdlib/src/collections/dict.mojo +++ b/stdlib/src/collections/dict.mojo @@ -957,3 +957,7 @@ struct OwnedKwargsDict[V: CollectionElement](Sized, CollectionElement): @always_inline("nodebug") fn _insert(inout self, owned key: Self.key_type, owned value: V): self._dict._insert(key^, value^) + + @always_inline("nodebug") + fn _insert(inout self, key: StringLiteral, owned value: V): + self._insert(String(key), value^) From b5c5d421b67315a605e03b634679530354e7aece Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 21:20:00 -0600 Subject: [PATCH 39/55] [stdlib] Avoid `import *` in tests (#36678) Change the tests to explicitly import what they use rather than using `from import *`. Upstream commit: [Internal Link] modular-orig-commit: d7ad4220c1e478180d3e300b39f8ad815ffbd8a8 --- examples/deviceinfo.mojo | 22 ++++++++++++++++++-- stdlib/test/builtin/test_bfloat16.mojo | 2 +- stdlib/test/builtin/test_dtype.mojo | 2 +- stdlib/test/builtin/test_hash.mojo | 2 +- stdlib/test/builtin/test_int.mojo | 2 +- stdlib/test/builtin/test_issue_1505.mojo | 2 +- stdlib/test/builtin/test_location.mojo | 2 +- stdlib/test/builtin/test_simd.mojo | 2 +- stdlib/test/builtin/test_slice.mojo | 2 +- stdlib/test/builtin/test_str.mojo | 2 +- stdlib/test/builtin/test_string.mojo | 8 ++++++- stdlib/test/builtin/test_string_literal.mojo | 8 ++++++- stdlib/test/builtin/test_stringref.mojo | 2 +- stdlib/test/collections/test_dict.mojo | 2 +- stdlib/test/collections/test_list.mojo | 2 +- stdlib/test/collections/test_optional.mojo | 2 +- stdlib/test/collections/test_set.mojo | 2 +- stdlib/test/collections/test_vector.mojo | 2 +- stdlib/test/memory/test_anypointer.mojo | 2 +- stdlib/test/memory/test_arc.mojo | 2 +- stdlib/test/os/path/test_exists.mojo | 2 +- stdlib/test/os/path/test_isdir.mojo | 2 +- stdlib/test/os/path/test_isfile.mojo | 2 +- stdlib/test/os/path/test_islink.mojo | 2 +- stdlib/test/os/test_listdir.mojo | 4 ++-- stdlib/test/os/test_stat.mojo | 4 ++-- stdlib/test/pathlib/test_pathlib.mojo | 4 ++-- stdlib/test/test_utils/__init__.mojo | 2 +- stdlib/test/utils/test_variant.mojo | 2 +- 29 files changed, 63 insertions(+), 33 deletions(-) diff --git a/examples/deviceinfo.mojo b/examples/deviceinfo.mojo index 62428a9b53..f2c79fa3e1 100644 --- a/examples/deviceinfo.mojo +++ b/examples/deviceinfo.mojo @@ -14,8 +14,26 @@ # This sample prints the current host system information using APIs from the # sys module. -from sys.info import * -from sys.info import _current_cpu, _current_target, _triple_attr +from sys.info import ( + _current_cpu, + _current_target, + _triple_attr, + os_is_linux, + os_is_macos, + os_is_windows, + has_sse4, + has_avx, + has_avx2, + has_avx512f, + has_vnni, + has_intel_amx, + has_neon, + is_apple_m1, + is_apple_m2, + is_apple_m3, + num_physical_cores, + num_logical_cores, +) def main(): diff --git a/stdlib/test/builtin/test_bfloat16.mojo b/stdlib/test/builtin/test_bfloat16.mojo index bc3b5f10b5..3b341e4ba7 100644 --- a/stdlib/test/builtin/test_bfloat16.mojo +++ b/stdlib/test/builtin/test_bfloat16.mojo @@ -15,7 +15,7 @@ from random import randn_float64 from sys.info import has_neon -from testing import * +from testing import assert_equal, assert_almost_equal def test_methods(): diff --git a/stdlib/test/builtin/test_dtype.mojo b/stdlib/test/builtin/test_dtype.mojo index b0fc366792..5e996d6677 100644 --- a/stdlib/test/builtin/test_dtype.mojo +++ b/stdlib/test/builtin/test_dtype.mojo @@ -14,7 +14,7 @@ from collections import Set -from testing import * +from testing import assert_equal, assert_false, assert_true fn test_stringable() raises: diff --git a/stdlib/test/builtin/test_hash.mojo b/stdlib/test/builtin/test_hash.mojo index 934b2c15b8..1bd5aac2d9 100644 --- a/stdlib/test/builtin/test_hash.mojo +++ b/stdlib/test/builtin/test_hash.mojo @@ -19,7 +19,7 @@ # specific. But for now they test behavior and reproducibility. from builtin.hash import _hash_simd -from testing import * +from testing import assert_equal, assert_not_equal, assert_true def same_low_bits(i1: Int, i2: Int, bits: Int = 4) -> Int: diff --git a/stdlib/test/builtin/test_int.mojo b/stdlib/test/builtin/test_int.mojo index 82b5046c20..c983817c19 100644 --- a/stdlib/test/builtin/test_int.mojo +++ b/stdlib/test/builtin/test_int.mojo @@ -12,7 +12,7 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from testing import * +from testing import assert_equal def test_constructors(): diff --git a/stdlib/test/builtin/test_issue_1505.mojo b/stdlib/test/builtin/test_issue_1505.mojo index aea2405573..8c1ef6eb20 100644 --- a/stdlib/test/builtin/test_issue_1505.mojo +++ b/stdlib/test/builtin/test_issue_1505.mojo @@ -16,7 +16,7 @@ from random import random_ui64 -from testing import * +from testing import assert_equal fn gen_perm() -> StaticIntTuple[64]: diff --git a/stdlib/test/builtin/test_location.mojo b/stdlib/test/builtin/test_location.mojo index ec5b730ecd..500230e460 100644 --- a/stdlib/test/builtin/test_location.mojo +++ b/stdlib/test/builtin/test_location.mojo @@ -13,7 +13,7 @@ # RUN: mojo --debug-level full %s from builtin._location import _SourceLocation -from testing import * +from testing import assert_equal def main(): diff --git a/stdlib/test/builtin/test_simd.mojo b/stdlib/test/builtin/test_simd.mojo index 60c767a9c9..c94d06b7a6 100644 --- a/stdlib/test/builtin/test_simd.mojo +++ b/stdlib/test/builtin/test_simd.mojo @@ -14,7 +14,7 @@ from sys.info import has_neon, simdwidthof -from testing import * +from testing import assert_equal, assert_not_equal, assert_true # CHECK-LABEL: test_cast diff --git a/stdlib/test/builtin/test_slice.mojo b/stdlib/test/builtin/test_slice.mojo index 1370bf88d5..0a405c0be5 100644 --- a/stdlib/test/builtin/test_slice.mojo +++ b/stdlib/test/builtin/test_slice.mojo @@ -12,7 +12,7 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s | FileCheck %s -from testing import * +from testing import assert_equal, assert_false # CHECK-LABEL: test_none_end_folds diff --git a/stdlib/test/builtin/test_str.mojo b/stdlib/test/builtin/test_str.mojo index 3e8dbd3c85..5694d97d22 100644 --- a/stdlib/test/builtin/test_str.mojo +++ b/stdlib/test/builtin/test_str.mojo @@ -12,7 +12,7 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from testing import * +from testing import assert_equal def test_str_none(): diff --git a/stdlib/test/builtin/test_string.mojo b/stdlib/test/builtin/test_string.mojo index 2cbfc5db2a..2da62facd5 100644 --- a/stdlib/test/builtin/test_string.mojo +++ b/stdlib/test/builtin/test_string.mojo @@ -16,7 +16,13 @@ from builtin.string import ( _calc_initial_buffer_size_int32, _calc_initial_buffer_size_int64, ) -from testing import * +from testing import ( + assert_equal, + assert_false, + assert_not_equal, + assert_raises, + assert_true, +) from utils import StringRef diff --git a/stdlib/test/builtin/test_string_literal.mojo b/stdlib/test/builtin/test_string_literal.mojo index 0299683f94..4473dadcc6 100644 --- a/stdlib/test/builtin/test_string_literal.mojo +++ b/stdlib/test/builtin/test_string_literal.mojo @@ -12,7 +12,13 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from testing import * +from testing import ( + assert_equal, + assert_not_equal, + assert_true, + assert_false, + assert_raises, +) def test_basics(): diff --git a/stdlib/test/builtin/test_stringref.mojo b/stdlib/test/builtin/test_stringref.mojo index fa4c62e890..bddfb751f4 100644 --- a/stdlib/test/builtin/test_stringref.mojo +++ b/stdlib/test/builtin/test_stringref.mojo @@ -12,7 +12,7 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from testing import * +from testing import assert_equal, assert_raises from utils import StringRef diff --git a/stdlib/test/collections/test_dict.mojo b/stdlib/test/collections/test_dict.mojo index f4afc36612..dadd2bf2c3 100644 --- a/stdlib/test/collections/test_dict.mojo +++ b/stdlib/test/collections/test_dict.mojo @@ -16,7 +16,7 @@ from collections import Optional from collections.dict import Dict, KeyElement, OwnedKwargsDict from test_utils import CopyCounter -from testing import * +from testing import assert_equal, assert_false, assert_raises, assert_true def test_dict_construction(): diff --git a/stdlib/test/collections/test_list.mojo b/stdlib/test/collections/test_list.mojo index de74a5286f..fc5364ce31 100644 --- a/stdlib/test/collections/test_list.mojo +++ b/stdlib/test/collections/test_list.mojo @@ -15,7 +15,7 @@ from collections import List from test_utils import CopyCounter, MoveCounter -from testing import * +from testing import assert_equal def test_mojo_issue_698(): diff --git a/stdlib/test/collections/test_optional.mojo b/stdlib/test/collections/test_optional.mojo index 1eff5ae14c..774af69f3c 100644 --- a/stdlib/test/collections/test_optional.mojo +++ b/stdlib/test/collections/test_optional.mojo @@ -14,7 +14,7 @@ from collections.optional import Optional, OptionalReg -from testing import * +from testing import assert_true, assert_false, assert_equal def test_basic(): diff --git a/stdlib/test/collections/test_set.mojo b/stdlib/test/collections/test_set.mojo index 74ef6409f7..4e94a06d7e 100644 --- a/stdlib/test/collections/test_set.mojo +++ b/stdlib/test/collections/test_set.mojo @@ -14,7 +14,7 @@ from collections.set import Set -from testing import * +from testing import assert_raises, assert_true, assert_false fn assert_equal[T: EqualityComparable](lhs: T, rhs: T) raises: diff --git a/stdlib/test/collections/test_vector.mojo b/stdlib/test/collections/test_vector.mojo index 406a782d84..b242b792f8 100644 --- a/stdlib/test/collections/test_vector.mojo +++ b/stdlib/test/collections/test_vector.mojo @@ -15,7 +15,7 @@ from collections.vector import InlinedFixedVector from test_utils import MoveCounter -from testing import * +from testing import assert_equal def test_inlined_fixed_vector(): diff --git a/stdlib/test/memory/test_anypointer.mojo b/stdlib/test/memory/test_anypointer.mojo index ce9377089f..52c0d0583f 100644 --- a/stdlib/test/memory/test_anypointer.mojo +++ b/stdlib/test/memory/test_anypointer.mojo @@ -14,7 +14,7 @@ from memory.anypointer import AnyPointer from test_utils import MoveCounter -from testing import * +from testing import assert_equal, assert_not_equal, assert_true struct MoveOnlyType(Movable): diff --git a/stdlib/test/memory/test_arc.mojo b/stdlib/test/memory/test_arc.mojo index c2748d1cfd..2e30349136 100644 --- a/stdlib/test/memory/test_arc.mojo +++ b/stdlib/test/memory/test_arc.mojo @@ -15,7 +15,7 @@ from collections import List from memory._arc import Arc -from testing import * +from testing import assert_equal, assert_false, assert_true def test_basic(): diff --git a/stdlib/test/os/path/test_exists.mojo b/stdlib/test/os/path/test_exists.mojo index 347e1f1c79..966f022f91 100644 --- a/stdlib/test/os/path/test_exists.mojo +++ b/stdlib/test/os/path/test_exists.mojo @@ -16,7 +16,7 @@ from os.path import exists, lexists from pathlib import Path, cwd -from testing import * +from testing import assert_true, assert_false def main(): diff --git a/stdlib/test/os/path/test_isdir.mojo b/stdlib/test/os/path/test_isdir.mojo index c539d2f310..b5a936f0fd 100644 --- a/stdlib/test/os/path/test_isdir.mojo +++ b/stdlib/test/os/path/test_isdir.mojo @@ -15,7 +15,7 @@ from os.path import isdir from pathlib import Path, cwd -from testing import * +from testing import assert_true, assert_false def main(): diff --git a/stdlib/test/os/path/test_isfile.mojo b/stdlib/test/os/path/test_isfile.mojo index 4e5b07152f..012a251b6d 100644 --- a/stdlib/test/os/path/test_isfile.mojo +++ b/stdlib/test/os/path/test_isfile.mojo @@ -15,7 +15,7 @@ from os.path import isfile from pathlib import Path -from testing import * +from testing import assert_true, assert_false def main(): diff --git a/stdlib/test/os/path/test_islink.mojo b/stdlib/test/os/path/test_islink.mojo index a63d15a72d..f8d55d5909 100644 --- a/stdlib/test/os/path/test_islink.mojo +++ b/stdlib/test/os/path/test_islink.mojo @@ -20,7 +20,7 @@ from os.path import isdir, islink from pathlib import Path from sys.param_env import env_get_string -from testing import * +from testing import assert_true, assert_false alias TEMP_DIR = env_get_string["TEMP_DIR"]() diff --git a/stdlib/test/os/test_listdir.mojo b/stdlib/test/os/test_listdir.mojo index 6737bd8504..945bdf6b3a 100644 --- a/stdlib/test/os/test_listdir.mojo +++ b/stdlib/test/os/test_listdir.mojo @@ -12,10 +12,10 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from os import * +from os import listdir from pathlib import Path -from testing import * +from testing import assert_true def test_listdir(): diff --git a/stdlib/test/os/test_stat.mojo b/stdlib/test/os/test_stat.mojo index 6e45452905..1e1a3d5ffb 100644 --- a/stdlib/test/os/test_stat.mojo +++ b/stdlib/test/os/test_stat.mojo @@ -12,10 +12,10 @@ # ===----------------------------------------------------------------------=== # # RUN: %mojo -debug-level full %s -from os import * +from os import stat from stat import S_ISREG -from testing import * +from testing import assert_not_equal, assert_true def main(): diff --git a/stdlib/test/pathlib/test_pathlib.mojo b/stdlib/test/pathlib/test_pathlib.mojo index 9f67761117..b19d5599f0 100644 --- a/stdlib/test/pathlib/test_pathlib.mojo +++ b/stdlib/test/pathlib/test_pathlib.mojo @@ -13,10 +13,10 @@ # REQUIRES: !windows # RUN: %mojo -debug-level full -D TEMP_FILE=%t %s -from pathlib import * +from pathlib import cwd, Path, DIR_SEPARATOR from sys.param_env import env_get_string -from testing import * +from testing import assert_true, assert_false, assert_equal, assert_not_equal alias TEMP_FILE = env_get_string["TEMP_FILE"]() diff --git a/stdlib/test/test_utils/__init__.mojo b/stdlib/test/test_utils/__init__.mojo index 291c1de528..976e8db514 100644 --- a/stdlib/test/test_utils/__init__.mojo +++ b/stdlib/test/test_utils/__init__.mojo @@ -4,5 +4,5 @@ # # ===----------------------------------------------------------------------=== # -from .test_utils import * +from .test_utils import libm_call from .types import CopyCounter, MoveCounter, MoveOnly diff --git a/stdlib/test/utils/test_variant.mojo b/stdlib/test/utils/test_variant.mojo index 709dee59e6..c9c4ec3048 100644 --- a/stdlib/test/utils/test_variant.mojo +++ b/stdlib/test/utils/test_variant.mojo @@ -15,7 +15,7 @@ from sys.ffi import _get_global from memory.unsafe import Pointer -from testing import * +from testing import assert_equal, assert_false, assert_true from utils.variant import Variant From 299ddd642f1624a1c289c2e7812764e4a484279d Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 21:20:27 -0600 Subject: [PATCH 40/55] [stdlib] Add `Dict.update()` (#36677) [Here](https://docs.python.org/3/library/stdtypes.html#dict.update) is the python documentation about this method. There should be another overload: `fn update(self, **kwargs: V)` which we can't do yet since it would only work with keys that are strings and we don't yet have conditional conformance (see [Internal Link] This is why we force the argument to be positional-only. If we allowed it to be passed by keyword, it would break when we will introduce the other overload of `Dict.update`. Upstream commit: [Internal Link] modular-orig-commit: d3fcf8609c6d97b4ea4d2bf866987d89e9e30a86 --- docs/changelog.md | 4 ++- stdlib/src/collections/dict.mojo | 10 ++++++ stdlib/test/collections/test_dict.mojo | 45 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 253a01a55a..1fb5fbef4d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -44,10 +44,12 @@ and tools. Please add any significant user-visible changes here. print(d) # prints `{'foo': [1, 2, 'something else'], 7: 'bar'}` ``` -- `List` collection now has a `pop(index)` API for removing an element +- `List` now has a `pop(index)` API for removing an element at a particular index. By default, `List.pop()` removes the last element in the list. +- `Dict` now has a `update()` method to update keys/values from another `Dict`. + ### πŸ¦‹ Changed - The behavior of `mojo build` when invoked without an output `-o` argument has diff --git a/stdlib/src/collections/dict.mojo b/stdlib/src/collections/dict.mojo index 40f81e563a..e826162147 100644 --- a/stdlib/src/collections/dict.mojo +++ b/stdlib/src/collections/dict.mojo @@ -638,6 +638,16 @@ struct Dict[K: KeyElement, V: CollectionElement](Sized, CollectionElement): 0, 0, Reference(self) ) + fn update(inout self, other: Self, /): + """Update the dictionary with the key/value pairs from other, overwriting existing keys. + The argument must be positional only. + + Args: + other: The dictionary to update from. + """ + for entry in other.items(): + self[entry[].key] = entry[].value + @staticmethod @always_inline fn _new_entries(reserved: Int) -> List[Optional[DictEntry[K, V]]]: diff --git a/stdlib/test/collections/test_dict.mojo b/stdlib/test/collections/test_dict.mojo index dadd2bf2c3..47bde31cbe 100644 --- a/stdlib/test/collections/test_dict.mojo +++ b/stdlib/test/collections/test_dict.mojo @@ -192,6 +192,48 @@ def test_dict_copy_calls_copy_constructor(): assert_equal(6, copy["a"].copy_count) +def test_dict_update_nominal(): + var orig = Dict[String, Int]() + orig["a"] = 1 + orig["b"] = 2 + + var new = Dict[String, Int]() + new["b"] = 3 + new["c"] = 4 + + orig.update(new) + + assert_equal(orig["a"], 1) + assert_equal(orig["b"], 3) + assert_equal(orig["c"], 4) + + +def test_dict_update_empty_origin(): + var orig = Dict[String, Int]() + var new = Dict[String, Int]() + new["b"] = 3 + new["c"] = 4 + + orig.update(new) + + assert_equal(orig["b"], 3) + assert_equal(orig["c"], 4) + + +def test_dict_update_empty_new(): + var orig = Dict[String, Int]() + orig["a"] = 1 + orig["b"] = 2 + + var new = Dict[String, Int]() + + orig.update(new) + + assert_equal(orig["a"], 1) + assert_equal(orig["b"], 2) + assert_equal(len(orig), 2) + + fn test[name: String, test_fn: fn () raises -> object]() raises: var name_val = name # FIXME(#26974): Can't pass 'name' directly. print("Test", name_val, "...", end="") @@ -223,6 +265,9 @@ def test_dict(): "test_dict_copy_calls_copy_constructor", test_dict_copy_calls_copy_constructor, ]() + test["test_dict_update_nominal", test_dict_update_nominal]() + test["test_dict_update_empty_origin", test_dict_update_empty_origin]() + test["test_dict_update_empty_new", test_dict_update_empty_new]() def test_taking_owned_kwargs_dict(owned kwargs: OwnedKwargsDict[Int]): From dacf20ea8a73093f86d182b5def3e5f0a248fc17 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 21:20:56 -0600 Subject: [PATCH 41/55] [stdlib] Replace `__init__ -> Self` use in `os` module (#36675) The `__init__ -> Self` is getting deprecated. Switch over to "normal" `__init__` constructors that accept `Self` as an `inout` argument instead within the `os` module. Upstream commit: [Internal Link] modular-orig-commit: 3382fe2636ae361e732a1b94a0c0dbdc4aef03a4 --- stdlib/src/os/_linux_aarch64.mojo | 38 ++++++++++++++--------------- stdlib/src/os/_linux_x86.mojo | 36 +++++++++++++--------------- stdlib/src/os/_macos.mojo | 40 +++++++++++++++---------------- 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/stdlib/src/os/_linux_aarch64.mojo b/stdlib/src/os/_linux_aarch64.mojo index f801bdf9f5..d25d1a3b9b 100644 --- a/stdlib/src/os/_linux_aarch64.mojo +++ b/stdlib/src/os/_linux_aarch64.mojo @@ -49,26 +49,24 @@ struct _c_stat(Stringable): var st_birthtimespec: _CTimeSpec # time of file creation(birth) var unused: StaticTuple[Int64, 2] # RESERVED: DO NOT USE! - fn __init__() -> Self: - return Self { - st_dev: 0, - st_mode: 0, - st_nlink: 0, - st_ino: 0, - st_uid: 0, - st_gid: 0, - __pad0: 0, - st_rdev: 0, - st_size: 0, - st_blksize: 0, - __pad1: 0, - st_blocks: 0, - st_atimespec: _CTimeSpec(), - st_mtimespec: _CTimeSpec(), - st_ctimespec: _CTimeSpec(), - st_birthtimespec: _CTimeSpec(), - unused: StaticTuple[Int64, 2](0, 0), - } + fn __init__(inout self): + self.st_dev = 0 + self.st_mode = 0 + self.st_nlink = 0 + self.st_ino = 0 + self.st_uid = 0 + self.st_gid = 0 + self.__pad0 = 0 + self.st_rdev = 0 + self.st_size = 0 + self.st_blksize = 0 + self.__pad1 = 0 + self.st_blocks = 0 + self.st_atimespec = _CTimeSpec() + self.st_mtimespec = _CTimeSpec() + self.st_ctimespec = _CTimeSpec() + self.st_birthtimespec = _CTimeSpec() + self.unused = StaticTuple[Int64, 2](0, 0) fn __str__(self) -> String: var res = String("{\n") diff --git a/stdlib/src/os/_linux_x86.mojo b/stdlib/src/os/_linux_x86.mojo index a94f66a54e..b0490520e1 100644 --- a/stdlib/src/os/_linux_x86.mojo +++ b/stdlib/src/os/_linux_x86.mojo @@ -48,25 +48,23 @@ struct _c_stat(Stringable): var st_birthtimespec: _CTimeSpec # time of file creation(birth) var unused: StaticTuple[Int64, 3] # RESERVED: DO NOT USE! - fn __init__() -> Self: - return Self { - st_dev: 0, - st_mode: 0, - st_nlink: 0, - st_ino: 0, - st_uid: 0, - st_gid: 0, - __pad0: 0, - st_rdev: 0, - st_size: 0, - st_blksize: 0, - st_blocks: 0, - st_atimespec: _CTimeSpec(), - st_mtimespec: _CTimeSpec(), - st_ctimespec: _CTimeSpec(), - st_birthtimespec: _CTimeSpec(), - unused: StaticTuple[Int64, 3](0, 0, 0), - } + fn __init__(inout self): + self.st_dev = 0 + self.st_mode = 0 + self.st_nlink = 0 + self.st_ino = 0 + self.st_uid = 0 + self.st_gid = 0 + self.__pad0 = 0 + self.st_rdev = 0 + self.st_size = 0 + self.st_blksize = 0 + self.st_blocks = 0 + self.st_atimespec = _CTimeSpec() + self.st_mtimespec = _CTimeSpec() + self.st_ctimespec = _CTimeSpec() + self.st_birthtimespec = _CTimeSpec() + self.unused = StaticTuple[Int64, 3](0, 0, 0) fn __str__(self) -> String: var res = String("{\n") diff --git a/stdlib/src/os/_macos.mojo b/stdlib/src/os/_macos.mojo index f0525e5477..374a79d904 100644 --- a/stdlib/src/os/_macos.mojo +++ b/stdlib/src/os/_macos.mojo @@ -51,27 +51,25 @@ struct _c_stat(Stringable): var st_lspare: Int32 # RESERVED: DO NOT USE! var st_qspare: StaticTuple[Int64, 2] # RESERVED: DO NOT USE! - fn __init__() -> Self: - return Self { - st_dev: 0, - st_mode: 0, - st_nlink: 0, - st_ino: 0, - st_uid: 0, - st_gid: 0, - st_rdev: 0, - st_atimespec: _CTimeSpec(), - st_mtimespec: _CTimeSpec(), - st_ctimespec: _CTimeSpec(), - st_birthtimespec: _CTimeSpec(), - st_size: 0, - st_blocks: 0, - st_blksize: 0, - st_flags: 0, - st_gen: 0, - st_lspare: 0, - st_qspare: StaticTuple[Int64, 2](0, 0), - } + fn __init__(inout self): + self.st_dev = 0 + self.st_mode = 0 + self.st_nlink = 0 + self.st_ino = 0 + self.st_uid = 0 + self.st_gid = 0 + self.st_rdev = 0 + self.st_atimespec = _CTimeSpec() + self.st_mtimespec = _CTimeSpec() + self.st_ctimespec = _CTimeSpec() + self.st_birthtimespec = _CTimeSpec() + self.st_size = 0 + self.st_blocks = 0 + self.st_blksize = 0 + self.st_flags = 0 + self.st_gen = 0 + self.st_lspare = 0 + self.st_qspare = StaticTuple[Int64, 2](0, 0) fn __str__(self) -> String: var res = String("{\n") From 991d6b54cd0df8bef03b09bffbc208fc45c57047 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Wed, 3 Apr 2024 21:30:47 -0600 Subject: [PATCH 42/55] [stdlib] Add `is` and `isnot` functions to `Optional` (#36674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just as the title says, it's a very common idiom in python, is even recommended by PEP8: https://peps.python.org/pep-0008/#programming-recommendations. Comparisons to singletons like `None` should always be done with is or is not, never the equality operators. Also, beware of writing if x when you really mean if x is not None – e.g. when testing whether a variable or argument that defaults to None was set to some other value. The other value might have a type (such as a container) that could be false in a boolean context! While the warning does not apply to Mojo, Python users are quite used to it and it's a small quality of life improvement which is inexpensive to support. Upstream commit: [Internal Link] modular-orig-commit: 0961d7b5804b574df61806b8c433471e72cfcd67 --- stdlib/src/collections/optional.mojo | 26 ++++++++++++++++++++++ stdlib/test/collections/test_optional.mojo | 18 +++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/stdlib/src/collections/optional.mojo b/stdlib/src/collections/optional.mojo index f218c0ede8..2bf494da43 100644 --- a/stdlib/src/collections/optional.mojo +++ b/stdlib/src/collections/optional.mojo @@ -147,6 +147,32 @@ struct Optional[T: CollectionElement](CollectionElement, Boolable): return self._value.get[T]()[] return default + fn __is__(self, other: NoneType) -> Bool: + """Return `True` if the Optional has no value. + + It allows you to use the following syntax: `if my_optional is None:` + + Args: + other: The value to compare to (None). + + Returns: + True if the Optional has no value and False otherwise. + """ + return not self.__bool__() + + fn __isnot__(self, other: NoneType) -> Bool: + """Return `True` if the Optional has a value. + + It allows you to use the following syntax: `if my_optional is not None:`. + + Args: + other: The value to compare to (None). + + Returns: + True if the Optional has a value and False otherwise. + """ + return self.__bool__() + fn __bool__(self) -> Bool: """Return true if the Optional has a value. diff --git a/stdlib/test/collections/test_optional.mojo b/stdlib/test/collections/test_optional.mojo index 774af69f3c..22a6858c34 100644 --- a/stdlib/test/collections/test_optional.mojo +++ b/stdlib/test/collections/test_optional.mojo @@ -71,6 +71,24 @@ def test_optional_reg_basic(): assert_true(True and val) +def test_optional_is(): + a = Optional(1) + assert_false(a is None) + + a = Optional[Int](None) + assert_true(a is None) + + +def test_optional_isnot(): + a = Optional(1) + assert_true(a is not None) + + a = Optional[Int](None) + assert_false(a is not None) + + def main(): test_basic() test_optional_reg_basic() + test_optional_is() + test_optional_isnot() From 44178a8f89921e1e702ea9c38a51a60a8aebb460 Mon Sep 17 00:00:00 2001 From: abdul dakkak Date: Wed, 3 Apr 2024 22:39:55 -0700 Subject: [PATCH 43/55] [Stdlib] Constrain the mask for shuffle to be within the right domain (#36719) Disallow negative and invalid values in the shuffle mask value. Closes [Internal Link] modular-orig-commit: dfac330b52860cf4418851b31807bfac0ccac2c8 --- stdlib/src/builtin/simd.mojo | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/stdlib/src/builtin/simd.mojo b/stdlib/src/builtin/simd.mojo index 8ab473ac0d..7b801eefde 100644 --- a/stdlib/src/builtin/simd.mojo +++ b/stdlib/src/builtin/simd.mojo @@ -1333,7 +1333,8 @@ struct SIMD[type: DType, size: Int = simdwidthof[type]()]( *mask: Int, output_size: Int = size ](self, other: Self) -> SIMD[type, output_size]: """Shuffles (also called blend) the values of the current vector with - the `other` value using the specified mask (permutation). + the `other` value using the specified mask (permutation). The mask values + must be within `2*len(self)`. Parameters: mask: The permutation to use in the shuffle. @@ -1368,6 +1369,10 @@ struct SIMD[type: DType, size: Int = simdwidthof[type]()]( @parameter fn fill[idx: Int](): alias val = mask[idx] + constrained[ + 0 <= val < 2 * size, + "invalid index in the shuffle operation", + ]() var ptr = __mlir_op.`pop.array.gep`( Pointer.address_of(array).address, idx.value ) @@ -1391,7 +1396,8 @@ struct SIMD[type: DType, size: Int = simdwidthof[type]()]( @always_inline("nodebug") fn shuffle[*mask: Int](self) -> Self: """Shuffles (also called blend) the values of the current vector with - the `other` value using the specified mask (permutation). + the `other` value using the specified mask (permutation). The mask values + must be within `2*len(self)`. Parameters: mask: The permutation to use in the shuffle. @@ -1405,7 +1411,8 @@ struct SIMD[type: DType, size: Int = simdwidthof[type]()]( @always_inline("nodebug") fn shuffle[*mask: Int](self, other: Self) -> Self: """Shuffles (also called blend) the values of the current vector with - the `other` value using the specified mask (permutation). + the `other` value using the specified mask (permutation). The mask values + must be within `2*len(self)`. Parameters: mask: The permutation to use in the shuffle. From 8900f36a54f414d00d38db73b5ea21ac192bae89 Mon Sep 17 00:00:00 2001 From: abdul dakkak Date: Wed, 3 Apr 2024 22:59:17 -0700 Subject: [PATCH 44/55] [Kernels] Rename apple and intel amx files in linalg package, NFC (#36727) modular-orig-commit: acf071cdc4ebba1ce41a3a059b5a2ef2077d896f --- stdlib/test/sys/test_has_intel_amx.mojo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/test/sys/test_has_intel_amx.mojo b/stdlib/test/sys/test_has_intel_amx.mojo index 9b8c526fb6..68cb8bc527 100644 --- a/stdlib/test/sys/test_has_intel_amx.mojo +++ b/stdlib/test/sys/test_has_intel_amx.mojo @@ -21,7 +21,7 @@ from sys.info import has_intel_amx, os_is_linux -from IntelAMX import init_intel_amx +from LinAlg.intel_amx import init_intel_amx # CHECK-LABEL: test_has_intel_amx From 07b7cedef1ab256ae4f655d998e91ead84e58a75 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Thu, 4 Apr 2024 09:14:54 -0600 Subject: [PATCH 45/55] [stdlib] Add move constructor to `Atomic` (#36718) Fixes [Internal Link] Upstream commit: [Internal Link] modular-orig-commit: 7654fad7242cdb9d88759558a1ba7ca360889144 --- stdlib/src/os/atomic.mojo | 9 +++++++++ stdlib/test/os/test_atomic.mojo | 22 +++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/stdlib/src/os/atomic.mojo b/stdlib/src/os/atomic.mojo index ec5c0ab592..a16a079c4c 100644 --- a/stdlib/src/os/atomic.mojo +++ b/stdlib/src/os/atomic.mojo @@ -57,6 +57,15 @@ struct Atomic[type: DType]: """ self.value = value + @always_inline + fn __moveinit__(inout self, owned existing: Self): + """Moves Constructor. + + Args: + existing: The existing Atomic. + """ + self.value = existing.value + @always_inline fn load(inout self) -> Scalar[type]: """Loads the current value from the atomic. diff --git a/stdlib/test/os/test_atomic.mojo b/stdlib/test/os/test_atomic.mojo index 563231e662..61c7c11ccc 100644 --- a/stdlib/test/os/test_atomic.mojo +++ b/stdlib/test/os/test_atomic.mojo @@ -13,6 +13,7 @@ # RUN: %mojo -debug-level full %s | FileCheck %s from os.atomic import Atomic +from testing import assert_equal # CHECK-LABEL: test_atomic @@ -90,6 +91,25 @@ fn test_atomic_floating_point(): print(atom.value) -fn main(): +def test_atomic_move_constructor(): + var atom: Atomic[DType.index] = 3 + var atom2 = atom^ + assert_equal(atom2.value, 3) + atom2 += 4 + assert_equal(atom2.value, 7) + atom2 -= 4 + assert_equal(atom2.value, 3) + atom2.max(0) + assert_equal(atom2.value, 3) + atom2.max(42) + assert_equal(atom2.value, 42) + atom2.min(3) + assert_equal(atom2.value, 3) + atom2.min(0) + assert_equal(atom2.value, 0) + + +def main(): test_atomic() test_atomic_floating_point() + test_atomic_move_constructor() From 4b5085d7afd712f85afdd29c0fa036ad48384142 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Thu, 4 Apr 2024 09:07:18 -0700 Subject: [PATCH 46/55] [mojo-lang] Enable `VariadicPack` for borrowed memory packs (#36680) This is the big one: `VariadicPack` is now ready to take on the heavy load for all non-register-passable packs, which notably includes `print` and `String.join`. This means deleting our old friend `_StringableTuple`, replacing the logic with stuff that is much simpler. These examples use capturing values, so switch the `VariadicPack.each` method to work with a capturing closure This patch fixes a ton of bugs, because variadic packs of memory-only type now work! modular-orig-commit: 436b86a7496642a6a8f4c4921ef7234436618123 --- stdlib/src/builtin/builtin_list.mojo | 2 +- stdlib/src/builtin/io.mojo | 76 ++++--------------------- stdlib/src/builtin/string.mojo | 28 +++++---- stdlib/test/memory/test_anypointer.mojo | 3 +- 4 files changed, 25 insertions(+), 84 deletions(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 9913d106d8..e26d643c79 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -538,7 +538,7 @@ struct VariadicPack[ return rebind[result_ref.mlir_ref_type](ref_elt) @always_inline - fn each[func: fn[T: element_trait] (T) -> None](self): + fn each[func: fn[T: element_trait] (T) capturing -> None](self): """Apply a function to each element of the pack in order. This applies the specified function (which must be parametric on the element type) to each element of the pack, from the first element to the last, passing diff --git a/stdlib/src/builtin/io.mojo b/stdlib/src/builtin/io.mojo index f8602a4810..ac461cd088 100644 --- a/stdlib/src/builtin/io.mojo +++ b/stdlib/src/builtin/io.mojo @@ -337,70 +337,6 @@ fn _put(x: DType): # ===----------------------------------------------------------------------=== # -struct _StringableTuple[*Ts: Stringable](Sized): - alias _type = __mlir_type[ - `!kgen.pack<:variadic<`, Stringable, `> `, Ts, `>` - ] - var storage: Self._type - - fn __init__(inout self, value: Self._type): - self.storage = value - - @staticmethod - fn _offset[i: Int]() -> Int: - constrained[i >= 0, "index must be positive"]() - - @parameter - if i == 0: - return 0 - else: - return _align_up( - Self._offset[i - 1]() - + _align_up(sizeof[Ts[i - 1]](), alignof[Ts[i - 1]]()), - alignof[Ts[i]](), - ) - - @always_inline - fn _print[i: Int](inout self, /, *, sep: StringLiteral = " "): - _put(sep) - _put(self._at[i]()) - - fn _at[i: Int](inout self) -> String: - alias offset = Self._offset[i]() - var i8ptr = Pointer.address_of(self).bitcast[Int8]().offset(offset) - return str(Reference(i8ptr[]).bitcast_element[Ts[i]]()[]) - - fn __len__(self) -> Int: - return len(VariadicList(Ts)) - - -@always_inline -fn _print_elements[ - T: Stringable, *Ts: Stringable -]( - first: T, - inout rest: _StringableTuple[Ts], - sep: StringLiteral = " ", - end: StringLiteral = "\n", - flush: Bool = False, -): - _put(str(first)) - - @parameter - fn each[i: Int](): - rest._print[i](sep=sep) - - unroll[each, len(VariadicList(Ts))]() - _put(end) - if flush: - _flush() - - -# ===----------------------------------------------------------------------=== # -# print -# ===----------------------------------------------------------------------=== # - - @no_inline fn print( *, sep: StringLiteral = " ", end: StringLiteral = "\n", flush: Bool = False @@ -441,7 +377,15 @@ fn print[ end: The String to write after printing the elements. flush: If set to true, then the stream is forcibly flushed. """ - var vals = _StringableTuple[Ts](rest) - _print_elements(first, vals, sep=sep, end=end) + _put(str(first)) + + @parameter + fn print_elt[T: Stringable](a: T): + _put(sep) + _put(a) + + rest.each[print_elt]() + + _put(end) if flush: _flush() diff --git a/stdlib/src/builtin/string.mojo b/stdlib/src/builtin/string.mojo index 269f210349..db3d8ce9bc 100644 --- a/stdlib/src/builtin/string.mojo +++ b/stdlib/src/builtin/string.mojo @@ -28,7 +28,7 @@ from utils import StringRef from utils.index import StaticIntTuple from utils.static_tuple import StaticTuple -from .io import _snprintf, _snprintf_scalar, _StringableTuple +from .io import _snprintf, _snprintf_scalar # ===----------------------------------------------------------------------===# # Utilties @@ -615,11 +615,11 @@ struct String(Sized, Stringable, IntableRaising, KeyElement, Boolable): curr += self + String(elems[i]) return curr - fn join[*Stringables: Stringable](self, *elems: *Stringables) -> String: + fn join[*Types: Stringable](self, *elems: *Types) -> String: """Joins string elements using the current string as a delimiter. Parameters: - Stringables: The Stringable types. + Types: The types of the elements. Args: elems: The input values. @@ -627,21 +627,19 @@ struct String(Sized, Stringable, IntableRaising, KeyElement, Boolable): Returns: The joined string. """ - alias types = VariadicList(Stringables) - alias count = len(types) - var args = _StringableTuple(elems) - - if count == 0: - return "" - - var result = args._at[0]() + var result: String = "" + var is_first = True @parameter - fn each[i: Int](): - result += self + args._at[i + 1]() - - unroll[each, count - 1]() + fn add_elt[T: Stringable](a: T): + if is_first: + is_first = False + else: + result += self + result += str(a) + + elems.each[add_elt]() return result fn _strref_dangerous(self) -> StringRef: diff --git a/stdlib/test/memory/test_anypointer.mojo b/stdlib/test/memory/test_anypointer.mojo index 52c0d0583f..19a14ce401 100644 --- a/stdlib/test/memory/test_anypointer.mojo +++ b/stdlib/test/memory/test_anypointer.mojo @@ -40,10 +40,9 @@ fn test_anypointer_of_move_only_type(): ptr.emplace_value(MoveOnlyType(42)) # CHECK: moved 42 var value = ptr.take_value() - # NOTE: Destructor is called before `print`. - # CHECK: deleted 42 # CHECK: value 42 print("value", value.value) + # CHECK: deleted 42 ptr.free() From 5ef3ed0b3b1f3867bfb2ca68355882eb95abed95 Mon Sep 17 00:00:00 2001 From: Stef Lindall Date: Thu, 4 Apr 2024 10:00:10 -0700 Subject: [PATCH 47/55] [mojo-stdlib] Parameterize `Reference` on `AddressSpace` (#36561) Fixes [Internal Link] - Adds an `address_space` parameter to `Reference` - Wires `address_space` through `Reference` methods - Updates `_LITRef` helper to speak `AddressSpace` types instead of an MLIR type --------- Co-authored-by: Chris Lattner modular-orig-commit: 887d6b06b1fa807fc4092fefafb627ed657bb524 --- stdlib/src/memory/memory.mojo | 4 +-- stdlib/src/memory/unsafe.mojo | 57 +++++++++++++++++++++++++++-------- stdlib/src/utils/variant.mojo | 4 +-- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/stdlib/src/memory/memory.mojo b/stdlib/src/memory/memory.mojo index 3604762f15..0cc9152742 100644 --- a/stdlib/src/memory/memory.mojo +++ b/stdlib/src/memory/memory.mojo @@ -445,14 +445,14 @@ fn stack_allocation[ count = count.value, _type = Pointer[type, address_space].pointer_type, alignment = alignment.value, - address_space = address_space.value().value, + address_space = address_space._value.value, ]() else: return __mlir_op.`pop.stack_allocation`[ count = count.value, _type = Pointer[type, address_space].pointer_type, alignment = alignment.value, - address_space = address_space.value().value, + address_space = address_space._value.value, ]() diff --git a/stdlib/src/memory/unsafe.mojo b/stdlib/src/memory/unsafe.mojo index ccd05eab31..6416a22434 100644 --- a/stdlib/src/memory/unsafe.mojo +++ b/stdlib/src/memory/unsafe.mojo @@ -432,7 +432,7 @@ struct _LITRef[ element_type: AnyType, elt_is_mutable: __mlir_type.i1, lifetime: AnyLifetime[elt_is_mutable].type, - addr_space: __mlir_type.index = Int(0).__mlir_index__(), + address_space: AddressSpace = AddressSpace.GENERIC, ]: alias type = __mlir_type[ `!lit.ref<`, @@ -440,7 +440,7 @@ struct _LITRef[ `, `, lifetime, `, `, - addr_space, + address_space._value.value, `>`, ] @@ -451,6 +451,7 @@ struct Reference[ type: AnyType, is_mutable: __mlir_type.i1, lifetime: AnyLifetime[is_mutable].type, + address_space: AddressSpace = AddressSpace.GENERIC, ]: """Defines a non-nullable safe reference. @@ -458,9 +459,12 @@ struct Reference[ type: Type of the underlying data. is_mutable: Whether the referenced data may be mutated through this. lifetime: The lifetime of the reference. + address_space: The address space of the referenced data. """ - alias mlir_ref_type = _LITRef[type, is_mutable, lifetime].type + alias mlir_ref_type = _LITRef[ + type, is_mutable, lifetime, address_space + ].type var value: Self.mlir_ref_type """The underlying MLIR reference.""" @@ -495,7 +499,7 @@ struct Reference[ # FIXME: This should be on Pointer, but can't due to AnyRefType vs AnyType # disagreement. @always_inline("nodebug") - fn get_unsafe_pointer(self) -> Pointer[type]: + fn get_unsafe_pointer(self) -> Pointer[type, address_space]: """Constructs a Pointer from a safe reference. Returns: @@ -504,7 +508,7 @@ struct Reference[ var ptr_with_trait = __mlir_op.`lit.ref.to_pointer`(self.value) # Work around AnyRefType vs AnyType. return __mlir_op.`pop.pointer.bitcast`[ - _type = Pointer[type].pointer_type + _type = Pointer[type, address_space].pointer_type ](ptr_with_trait) @always_inline("nodebug") @@ -522,7 +526,7 @@ struct Reference[ @always_inline("nodebug") fn bitcast_element[ new_element_type: AnyType - ](self) -> Reference[new_element_type, is_mutable, lifetime]: + ](self) -> Reference[new_element_type, is_mutable, lifetime, address_space]: """Cast the reference to one of another element type, but the same lifetime, mutability, and address space. @@ -536,10 +540,18 @@ struct Reference[ # to KGEN pointer. var kgen_ptr = __mlir_op.`lit.ref.to_pointer`(self.value) var dest_ptr = __mlir_op.`pop.pointer.bitcast`[ - _type = __mlir_type[`!kgen.pointer<`, new_element_type, `>`] + _type = __mlir_type[ + `!kgen.pointer<`, + new_element_type, + `,`, + address_space._value.value, + `>`, + ] ](kgen_ptr) return __mlir_op.`lit.ref.from_pointer`[ - _type = _LITRef[new_element_type, is_mutable, lifetime].type + _type = _LITRef[ + new_element_type, is_mutable, lifetime, address_space + ].type ](dest_ptr) fn destroy_element_unsafe(self): @@ -554,11 +566,30 @@ struct Reference[ is_mutable, "cannot use 'unsafe_destroy_element' on immutable references", ]() + + # This method can only work on address space 0, because the __del__ + # method that we need to invoke will take 'self' in address space zero. + constrained[ + address_space == AddressSpace.GENERIC, + "cannot use 'destroy_element_unsafe' on arbitrary address spaces", + ]() + # Project to an owned raw pointer, allowing the compiler to know it is to # be destroyed. - # TODO: Use AnyPointer, but it requires a Movable element. var kgen_ptr = __mlir_op.`lit.ref.to_pointer`(self.value) - _ = __get_address_as_owned_value(kgen_ptr) + + # Bitcast to address space zero since the inserted __del__ call will only + # work with address space zero. + var dest_ptr = __mlir_op.`pop.pointer.bitcast`[ + _type = __mlir_type[ + `!kgen.pointer<`, + type, + `>`, + ] + ](kgen_ptr) + + # TODO: Use AnyPointer, but it requires a Movable element. + _ = __get_address_as_owned_value(dest_ptr) # FIXME: This should be a method on Reference, it is placed here because we need @@ -601,7 +632,7 @@ struct Pointer[ """ alias pointer_type = __mlir_type[ - `!kgen.pointer<`, type, `,`, address_space.value().value, `>` + `!kgen.pointer<`, type, `,`, address_space._value.value, `>` ] var address: Self.pointer_type @@ -611,7 +642,7 @@ struct Pointer[ type, __mlir_attr.`1: i1`, __mlir_attr.`#lit.lifetime<1>: !lit.lifetime<1>`, - address_space.value().value, + address_space, ].type @always_inline("nodebug") @@ -1073,7 +1104,7 @@ struct DTypePointer[ `!kgen.pointer,`, - address_space.value().value, + address_space._value.value, `>`, ], ): diff --git a/stdlib/src/utils/variant.mojo b/stdlib/src/utils/variant.mojo index 66c473cfab..6daa969745 100644 --- a/stdlib/src/utils/variant.mojo +++ b/stdlib/src/utils/variant.mojo @@ -160,9 +160,7 @@ struct Variant[*Ts: CollectionElement](CollectionElement): fn _get_state[ is_mut: __mlir_type.i1, lt: __mlir_type[`!lit.lifetime<`, is_mut, `>`] - ](self: _LITRef[Self, is_mut, lt, Int(0).value].type) -> Reference[ - Int8, is_mut, lt - ]: + ](self: _LITRef[Self, is_mut, lt].type) -> Reference[Int8, is_mut, lt]: return ( Reference(self) .bitcast_element[Int8]() From 52720283aca61d7734a997de576df6fb52e94b26 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Thu, 4 Apr 2024 10:19:15 -0700 Subject: [PATCH 48/55] [mojo-lang] Document variadic packs work in changelog. (#36781) modular-orig-commit: 1b22c60e8429217ec8f3e37a28e81f970a04a6c2 --- docs/changelog.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 1fb5fbef4d..5df61b88e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,21 @@ and tools. Please add any significant user-visible changes here. ### ⭐️ New +- Heterogenous variadic pack arguments now work reliably even with memory types, + and have a more convenient API to use, as defined on the `VariadicPack` type. + For example, a simplified version of `print` can be implemented as: + + ```mojo + fn print[T: Stringable, *Ts: Stringable](first: T, *rest: *Ts): + print_string(str(first)) + + @parameter + fn print_elt[T: Stringable](a: T): + print_string(" ") + print_string(a) + rest.each[print_elt]() + ``` + - The `sys` module now contains an `exit` function that would exit a Mojo program with the specified error code. From f9ce2fb128cd23ff656f79c20cac25c4f34cc3a7 Mon Sep 17 00:00:00 2001 From: James Decker Date: Thu, 4 Apr 2024 16:53:58 -0600 Subject: [PATCH 49/55] [mblack] Fix tokenization error in mblack (#36826) We goofed when adding backtick support and caused mblack to parse `!` incorrectly. Running mblack on the repo after making this change also caused a bunch of other files to need formatting, so this PR includes those as well. Fixes [Internal Link] modular-orig-commit: daad7e9267589460a01cdf9612d8d34134479de9 --- docs/changelog.md | 3 +++ stdlib/src/builtin/int.mojo | 2 +- stdlib/src/builtin/simd.mojo | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 5df61b88e9..502d60bd97 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -102,3 +102,6 @@ and tools. Please add any significant user-visible changes here. - [#1924](https://github.com/modularml/mojo/issues/1924) JIT debugging on Mac has been fixed. + +- [#1963](https://github.com/modularml/mojo/issues/1963) `a!=0` is now parsed + and formatted correctly by `mojo format`. diff --git a/stdlib/src/builtin/int.mojo b/stdlib/src/builtin/int.mojo index 297e3e2ef1..ce8859e921 100644 --- a/stdlib/src/builtin/int.mojo +++ b/stdlib/src/builtin/int.mojo @@ -616,7 +616,7 @@ struct Int(Intable, Stringable, KeyElement, Boolable): var x = self var n = rhs while n > 0: - if n&1 != 0: + if n & 1 != 0: res *= x x *= x n >>= 1 diff --git a/stdlib/src/builtin/simd.mojo b/stdlib/src/builtin/simd.mojo index 7b801eefde..7c23117ed2 100644 --- a/stdlib/src/builtin/simd.mojo +++ b/stdlib/src/builtin/simd.mojo @@ -2225,7 +2225,7 @@ fn _pow[ var x = lhs[i] var n = rhs[i] while n > 0: - if n&1 != 0: + if n & 1 != 0: res *= x x *= x n >>= 1 From 0c02f93b15c93be698e7792ef82ab4162b0d2929 Mon Sep 17 00:00:00 2001 From: Scott Main Date: Thu, 4 Apr 2024 16:40:17 -0700 Subject: [PATCH 50/55] [docs] Delete playground deprecation notice (#36842) That playground is gone now. modular-orig-commit: 097e1bd349e1a181cf1526732d9f4f8ec0c3071a --- docs/manual/get-started/index.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/manual/get-started/index.md b/docs/manual/get-started/index.md index 3fe41ff422..0f226917f7 100644 --- a/docs/manual/get-started/index.md +++ b/docs/manual/get-started/index.md @@ -188,13 +188,6 @@ brew upgrade modular Instead of downloading the Mojo SDK, you can also experiment with Mojo in our online [Playground](/mojo/playground). -:::note - -The older [JupyterLab-based Playground](https://playground.modular.com) is still -available until March 20th. - -::: - ### What to expect The Mojo Playground is a simple online editor where you can test out Mojo From b2377c741c73f88229cf643518081349df2ffacc Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Thu, 4 Apr 2024 23:32:01 -0700 Subject: [PATCH 51/55] [mojo-lang] Move testsuite off `AnyRegType` packs. (#36884) This moves the testsuite off of `AnyRegType` packs and onto `AnyType` packs as an initial step to making `AnyRegType` packs invalid. modular-orig-commit: 79b5189ec41b113965c829837258680b9dcdbbec --- stdlib/src/builtin/builtin_list.mojo | 1 - 1 file changed, 1 deletion(-) diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index e26d643c79..4dab05f7f6 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -419,7 +419,6 @@ struct VariadicPack[ lifetime: AnyLifetime[elt_is_mutable].type, element_trait: _AnyTypeMetaType, *element_types: element_trait, - # TODO: Add address_space when Reference supports it. ](Sized): """A utility class to access variadic pack arguments and provide an API for doing things with them. From a49ff9cd7814b9f288221311aaed92e009eb017d Mon Sep 17 00:00:00 2001 From: Rob Parolin Date: Fri, 5 Apr 2024 06:48:33 -0700 Subject: [PATCH 52/55] [stdlib] Adding AddressSpace support to AnyPointer. (#36552) Adding an AddressSpace parameter to AnyPointer to make it usable in GPU contexts. This change includes fixes to the Dict implementation that resolve compilation failures. Also, fixes to Mojo parser tests that are validating generated mlir that changed due to the addition of the AddressSpace parameter. Closes #31636 modular-orig-commit: 8ecb40292c2113cd19fc99c590910a0952b542c4 --- stdlib/src/collections/dict.mojo | 34 +++++++++++++++++++------ stdlib/src/memory/anypointer.mojo | 33 ++++++++++++++++-------- stdlib/test/memory/test_anypointer.mojo | 10 ++++++++ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/stdlib/src/collections/dict.mojo b/stdlib/src/collections/dict.mojo index e826162147..3311edbfb9 100644 --- a/stdlib/src/collections/dict.mojo +++ b/stdlib/src/collections/dict.mojo @@ -51,6 +51,7 @@ struct _DictEntryIter[ V: CollectionElement, dict_mutability: __mlir_type.`i1`, dict_lifetime: AnyLifetime[dict_mutability].type, + address_space: AddressSpace = AddressSpace.GENERIC, ]: """Iterator over immutable DictEntry references. @@ -59,6 +60,7 @@ struct _DictEntryIter[ V: The value type of the elements in the dictionary. dict_mutability: Whether the reference to the dictionary is mutable. dict_lifetime: The lifetime of the List + address_space: the address_space of the list """ alias imm_dict_lifetime = __mlir_attr[ @@ -101,6 +103,7 @@ struct _DictKeyIter[ V: CollectionElement, dict_mutability: __mlir_type.`i1`, dict_lifetime: AnyLifetime[dict_mutability].type, + address_space: AddressSpace = AddressSpace.GENERIC, ]: """Iterator over immutable Dict key references. @@ -109,14 +112,21 @@ struct _DictKeyIter[ V: The value type of the elements in the dictionary. dict_mutability: Whether the reference to the vector is mutable. dict_lifetime: The lifetime of the List + address_space: The address space of the List """ alias imm_dict_lifetime = __mlir_attr[ `#lit.lifetime.mutcast<`, dict_lifetime, `> : !lit.lifetime<1>` ] - alias ref_type = Reference[K, __mlir_attr.`0: i1`, Self.imm_dict_lifetime] + alias ref_type = Reference[ + K, __mlir_attr.`0: i1`, Self.imm_dict_lifetime, address_space + ] - var iter: _DictEntryIter[K, V, dict_mutability, dict_lifetime] + alias dict_entry_iter = _DictEntryIter[ + K, V, dict_mutability, dict_lifetime, address_space + ] + + var iter: Self.dict_entry_iter fn __iter__(self) -> Self: return self @@ -126,9 +136,13 @@ struct _DictKeyIter[ var mlir_ptr = __mlir_op.`lit.ref.to_pointer`( Reference(entry_ref[].key).value ) - var key_ptr = AnyPointer[K] { + var key_ptr = AnyPointer[ + K, address_space = Self.dict_entry_iter.address_space + ] { value: __mlir_op.`pop.pointer.bitcast`[ - _type = AnyPointer[K].pointer_type + _type = AnyPointer[ + K, address_space = Self.dict_entry_iter.address_space + ].pointer_type ](mlir_ptr) } return __mlir_op.`lit.ref.from_pointer`[ @@ -145,6 +159,7 @@ struct _DictValueIter[ V: CollectionElement, dict_mutability: __mlir_type.`i1`, dict_lifetime: AnyLifetime[dict_mutability].type, + address_space: AddressSpace = AddressSpace.GENERIC, ]: """Iterator over Dict value references. These are mutable if the dict is mutable. @@ -154,11 +169,14 @@ struct _DictValueIter[ V: The value type of the elements in the dictionary. dict_mutability: Whether the reference to the vector is mutable. dict_lifetime: The lifetime of the List + address_space: The address space of the List """ - alias ref_type = Reference[V, dict_mutability, dict_lifetime] + alias ref_type = Reference[V, dict_mutability, dict_lifetime, address_space] - var iter: _DictEntryIter[K, V, dict_mutability, dict_lifetime] + var iter: _DictEntryIter[ + K, V, dict_mutability, dict_lifetime, address_space + ] fn __iter__(self) -> Self: return self @@ -168,9 +186,9 @@ struct _DictValueIter[ var mlir_ptr = __mlir_op.`lit.ref.to_pointer`( Reference(entry_ref[].value).value ) - var value_ptr = AnyPointer[V] { + var value_ptr = AnyPointer[V, address_space] { value: __mlir_op.`pop.pointer.bitcast`[ - _type = AnyPointer[V].pointer_type + _type = AnyPointer[V, address_space].pointer_type ](mlir_ptr) } return __mlir_op.`lit.ref.from_pointer`[ diff --git a/stdlib/src/memory/anypointer.mojo b/stdlib/src/memory/anypointer.mojo index 251f1275a4..f06c0c8fd6 100644 --- a/stdlib/src/memory/anypointer.mojo +++ b/stdlib/src/memory/anypointer.mojo @@ -23,20 +23,24 @@ from sys.info import alignof, sizeof from sys.intrinsics import _mlirtype_is_eq from memory.memory import _free, _malloc +from memory.unsafe import _LITRef @register_passable("trivial") -struct AnyPointer[T: Movable]( - Boolable, CollectionElement, Stringable, Intable, EqualityComparable -): +struct AnyPointer[ + T: Movable, address_space: AddressSpace = AddressSpace.GENERIC +](Boolable, CollectionElement, Stringable, Intable, EqualityComparable): """This is a pointer type that can point to any generic value that is movable. Parameters: T: The pointer element type, which must be movable. + address_space: The address space associated with the AnyPointer allocated memory. """ - alias pointer_type = __mlir_type[`!kgen.pointer<`, T, `>`] + alias pointer_type = __mlir_type[ + `!kgen.pointer<`, T, `,`, address_space._value.value, `>` + ] """The underlying pointer type.""" var value: Self.pointer_type """The underlying pointer.""" @@ -76,7 +80,11 @@ struct AnyPointer[T: Movable]( The pointer to the newly allocated array. """ return Self.__from_index( - int(_malloc[Int8](sizeof[T]() * count, alignment=alignof[T]())) + int( + _malloc[Int8, address_space=address_space]( + sizeof[T]() * count, alignment=alignof[T]() + ) + ) ) @staticmethod @@ -97,10 +105,12 @@ struct AnyPointer[T: Movable]( @always_inline fn free(self): """Free the memory referenced by the pointer.""" - Pointer[Int8].__from_index(int(self)).free() + Pointer[Int8, address_space=address_space].__from_index( + int(self) + ).free() @always_inline - fn bitcast[new_type: Movable](self) -> AnyPointer[new_type]: + fn bitcast[new_type: Movable](self) -> AnyPointer[new_type, address_space]: """Bitcasts the pointer to a different type. Parameters: @@ -113,10 +123,10 @@ struct AnyPointer[T: Movable]( @parameter if _mlirtype_is_eq[T, new_type](): - return rebind[AnyPointer[new_type]](self) + return rebind[AnyPointer[new_type, address_space]](self) return __mlir_op.`pop.pointer.bitcast`[ - _type = AnyPointer[new_type].pointer_type + _type = AnyPointer[new_type, address_space].pointer_type ](self.value) @always_inline @@ -312,7 +322,10 @@ struct AnyPointer[T: Movable]( # an immortal mutable lifetime, since we can't come up with a meaningful # lifetime for them anyway. alias mlir_ref_type = Reference[ - T, __mlir_attr.`1: i1`, __mlir_attr.`#lit.lifetime<1>: !lit.lifetime<1>` + T, + __mlir_attr.`1: i1`, + __mlir_attr.`#lit.lifetime<1>: !lit.lifetime<1>`, + address_space, ].mlir_ref_type @always_inline diff --git a/stdlib/test/memory/test_anypointer.mojo b/stdlib/test/memory/test_anypointer.mojo index 19a14ce401..63606cb47e 100644 --- a/stdlib/test/memory/test_anypointer.mojo +++ b/stdlib/test/memory/test_anypointer.mojo @@ -135,6 +135,14 @@ def test_comparisons(): p1.free() +def test_anypointer_address_space(): + var p1 = AnyPointer[Int, AddressSpace(0)].alloc(1) + p1.free() + + var p2 = AnyPointer[Int, AddressSpace.GENERIC].alloc(1) + p2.free() + + def main(): test_address_of() @@ -148,3 +156,5 @@ def main(): test_anypointer_string() test_eq() test_comparisons() + + test_anypointer_address_space() From ec474378216a05e5a70c743303b3695ca3ea8a95 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Fri, 5 Apr 2024 09:22:35 -0700 Subject: [PATCH 53/55] [mojo-lang] Reject `AnyRegType` variadic packs, pushing to `AnyType` (#36906) `AnyRegType` variadic packs are a legacy feature that predated traits (and possibly memory only types in general). We are unifying on a single consistent representation built around `VariadicPack`. This takes the step of banning the old `AnyRegType` form in the parser. That said, we still have a few uses of the old form in the tree, so it keeps them alive when explicitly declared "borrowed". This will allow us to work through removing/replacing these cases incrementally. modular-orig-commit: 8e7642b4396f704020adabf48c2e7dfb01ec6e9d --- docs/changelog.md | 11 +++++++++++ stdlib/src/builtin/builtin_list.mojo | 2 +- stdlib/src/builtin/io.mojo | 9 +++++++-- stdlib/src/builtin/string.mojo | 5 ++++- stdlib/src/builtin/tuple.mojo | 2 +- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 502d60bd97..4fa3ee9248 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -80,6 +80,17 @@ and tools. Please add any significant user-visible changes here. ### ❌ Removed +- Support for "register only" variadic packs has been removed. Instead of + `AnyRegType`, please upgrade your code to `AnyType` in examples like this: + + ```mojo + fn [*Types: AnyRegType](*args: *Ts): ... + ``` + + This move gives you access to nicer API and has the benefit of being memory + safe and correct for non-trivial types. If you need specific APIs on the + types, please use the correct trait bound instead of `AnyType`. + - `List.pop_back()` has been removed. Use `List.pop()` instead which defaults to popping the last element in the list. diff --git a/stdlib/src/builtin/builtin_list.mojo b/stdlib/src/builtin/builtin_list.mojo index 4dab05f7f6..56cdf5ac8e 100644 --- a/stdlib/src/builtin/builtin_list.mojo +++ b/stdlib/src/builtin/builtin_list.mojo @@ -36,7 +36,7 @@ struct ListLiteral[*Ts: AnyRegType](Sized): """The underlying storage for the list.""" @always_inline("nodebug") - fn __init__(inout self, *args: *Ts): + fn __init__(inout self, borrowed *args: *Ts): """Construct the list literal from the given values. Args: diff --git a/stdlib/src/builtin/io.mojo b/stdlib/src/builtin/io.mojo index ac461cd088..7dfd4b0765 100644 --- a/stdlib/src/builtin/io.mojo +++ b/stdlib/src/builtin/io.mojo @@ -99,7 +99,7 @@ fn _flush(): @no_inline -fn _printf[*types: AnyRegType](fmt: StringLiteral, *arguments: *types): +fn _printf[*types: AnyRegType](fmt: StringLiteral, borrowed *arguments: *types): with _fdopen(_fdopen.STDOUT) as fd: _ = __mlir_op.`pop.external_call`[ func = "KGEN_CompilerRT_fprintf".value, @@ -121,7 +121,12 @@ fn _printf[*types: AnyRegType](fmt: StringLiteral, *arguments: *types): @no_inline fn _snprintf[ *types: AnyRegType -](str: Pointer[Int8], size: Int, fmt: StringLiteral, *arguments: *types) -> Int: +]( + str: Pointer[Int8], + size: Int, + fmt: StringLiteral, + borrowed *arguments: *types, +) -> Int: """Writes a format string into an output pointer. Args: diff --git a/stdlib/src/builtin/string.mojo b/stdlib/src/builtin/string.mojo index db3d8ce9bc..f282c37330 100644 --- a/stdlib/src/builtin/string.mojo +++ b/stdlib/src/builtin/string.mojo @@ -1088,7 +1088,10 @@ struct String(Sized, Stringable, IntableRaising, KeyElement, Boolable): fn _vec_fmt[ *types: AnyRegType ]( - str: AnyPointer[Int8], size: Int, fmt: StringLiteral, *arguments: *types + str: AnyPointer[Int8], + size: Int, + fmt: StringLiteral, + borrowed *arguments: *types, ) -> Int: return _snprintf(rebind[Pointer[Int8]](str), size, fmt, arguments) diff --git a/stdlib/src/builtin/tuple.mojo b/stdlib/src/builtin/tuple.mojo index 1c94d58ad4..068a9195bd 100644 --- a/stdlib/src/builtin/tuple.mojo +++ b/stdlib/src/builtin/tuple.mojo @@ -37,7 +37,7 @@ struct Tuple[*Ts: AnyRegType](Sized, CollectionElement): """The underlying storage for the tuple.""" @always_inline("nodebug") - fn __init__(*args: *Ts) -> Self: + fn __init__(borrowed *args: *Ts) -> Self: """Construct the tuple. Args: From 55ff418fa1b090458ff599a5b089dbe98511702d Mon Sep 17 00:00:00 2001 From: Jack Clayton Date: Fri, 5 Apr 2024 13:35:50 -0500 Subject: [PATCH 54/55] [Docs] Add section to style guide on inheriting Python inconsistencies (#36908) To avoid bikeshedding on the topic and have a place to point users to whenever the discussion comes up. modular-orig-commit: d35bcddaea49e7fcaa4ec5ec3ef7f88aa7b3ca42 --- stdlib/docs/style-guide.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stdlib/docs/style-guide.md b/stdlib/docs/style-guide.md index a7aff2e206..bf7b167727 100644 --- a/stdlib/docs/style-guide.md +++ b/stdlib/docs/style-guide.md @@ -121,6 +121,13 @@ struct MyStruct(Sized, Stringable): ## Code conventions +### Python Standard Library + +We want to be a good member of the Python family and aim to become a full +superset, so we inherit naming from the Python standard library, including any +inconsistencies. These naming inconsistencies are the only exceptions to the +naming conventions outlined below. + ### Identifier naming conventions There are several ways to capitalize and separate words, known as "case From f06a94017659199fb33b665fc34ca7742ee939b9 Mon Sep 17 00:00:00 2001 From: Joe Loser Date: Fri, 5 Apr 2024 10:20:51 -0600 Subject: [PATCH 55/55] [stdlib] Fixup some internal sync issues The `pop()` function got refactored after an internal sync, causing some issues. Separately, the `moveinit` function for `atomic` got duplicated during the internal sync. Remove the duplicate definitions so the `stdlib` can be packaged and work properly again. --- stdlib/src/collections/list.mojo | 25 ------------------------- stdlib/src/os/atomic.mojo | 9 --------- 2 files changed, 34 deletions(-) diff --git a/stdlib/src/collections/list.mojo b/stdlib/src/collections/list.mojo index d28efc5b6d..21387a5cdd 100644 --- a/stdlib/src/collections/list.mojo +++ b/stdlib/src/collections/list.mojo @@ -250,31 +250,6 @@ struct List[T: CollectionElement](CollectionElement, Sized): self._realloc(self.capacity // 2) return ret_val^ - @always_inline - fn pop(inout self, i: Int = -1) -> T: - """Pops a value from the list at the given index. - - Args: - i: The index of the value to pop. - - Returns: - The popped value. - """ - debug_assert(-self.size <= i < self.size, "pop index out of range") - - var normalized_idx = i - if i < 0: - normalized_idx += len(self) - - var ret_val = (self.data + normalized_idx).take_value() - for j in range(normalized_idx + 1, self.size): - (self.data + j).move_into(self.data + j - 1) - self.size -= 1 - if self.size * 4 < self.capacity: - if self.capacity > 1: - self._realloc(self.capacity // 2) - return ret_val^ - @always_inline fn reserve(inout self, new_capacity: Int): """Reserves the requested capacity. diff --git a/stdlib/src/os/atomic.mojo b/stdlib/src/os/atomic.mojo index 8eee75bf3b..a16a079c4c 100644 --- a/stdlib/src/os/atomic.mojo +++ b/stdlib/src/os/atomic.mojo @@ -75,15 +75,6 @@ struct Atomic[type: DType]: """ return self.fetch_add(0) - @always_inline - fn __moveinit__(inout self, owned existing: Self): - """Moves Constructor. - - Args: - existing: The existing Atomic. - """ - self.value = existing.value - @staticmethod @always_inline fn _fetch_add(