Skip to content

Commit 82970e5

Browse files
andrejandrej
authored
Merge d442395 into 003b49d 
2 parents 003b49d + d442395 commit 82970e5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+5701
-169
lines changed

‎.github/workflows/ci-lint.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
if: success() || failure()
2929
shell: bash
3030
run: |
31+
reuse download --all
3132
reuse lint
3233
3334
- name: Format Python

‎.github/workflows/test-examples.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Test Example Applications
5+
6+
on:
7+
workflow_dispatch:
8+
pull_request:
9+
push:
10+
branches:
11+
- main
12+
- devel
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
Test:
20+
runs-on: [self-hosted, docker]
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
env:
25+
HOME: /workspace
26+
27+
- name: Prerequisites
28+
uses: ./.github/actions/prereqs
29+
with:
30+
env_name: ci_env
31+
32+
- name: Install additional requirements for examples
33+
shell: bash
34+
env:
35+
HOME: /workspace
36+
run: |
37+
set -euxo pipefail
38+
source ci_env/bin/activate
39+
pip install -r requirements_examples.txt
40+
41+
- name: Build examples
42+
uses: ./.github/actions/build
43+
with:
44+
env_name: ci_env
45+
cmake_extra_flags: "-DBUILD_APPLICATIONS=ON -DBUILD_EXAMPLES=OFF"
46+
47+
- name: Run examples
48+
id: test
49+
continue-on-error: true
50+
uses: ./.github/actions/test
51+
with:
52+
env_name: ci_env
53+
test_flags: "--log tests.log --csv-latest tests_latest.csv"
54+
55+
- name: Commit example test results
56+
uses: ./.github/actions/commit_results
57+
id: commit_results
58+
with:
59+
dir: examples
60+
pretty_flags: "--metric='TTFT (mean)' --metric='TPS (mean)' -o pretty.md"
61+
62+
- name: Upload test results
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: results-examples
66+
path: ${{ steps.commit_results.outputs.results_dir }}
67+
retention-days: 14
68+
69+
- name: Fail workflow if examples failed
70+
if: steps.test.outcome == 'failure'
71+
run: exit 1

‎CMakeLists.txt‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,9 @@ endif()
6060
set(IRONCLAD_MLIR_AIE_RUNTIME_DIR "${IRONCLAD_MLIR_AIE_DIR}/runtime_lib/x86_64/test_lib/" CACHE PATH "Path to built/installed test_lib from MLIR_AIE")
6161

6262
# More Options
63-
option(BUILD_EXAMPLES "Build examples" ON)
64-
option(EXTENSIVE_TESTING "Generate a larger number of examples and tests" OFF)
63+
option(BUILD_EXAMPLES "Build operators" ON)
64+
option(BUILD_APPLICATIONS "Build example applications" OFF)
65+
option(EXTENSIVE_TESTING "Generate a larger number of operators and tests" OFF)
6566

6667
message(STATUS "IRONCLAD_PEANO_DIR: ${IRONCLAD_PEANO_DIR}")
6768
message(STATUS "IRONCLAD_MLIR_AIE_DIR: ${IRONCLAD_MLIR_AIE_DIR}")
@@ -95,8 +96,12 @@ add_subdirectory(aie_kernels)
9596
# Add examples
9697
# Pass the architecure to support aie2p
9798
if (BUILD_EXAMPLES)
98-
include(cmake/IroncladExamples.cmake)
9999
add_subdirectory(example)
100100
endif()
101101

102+
# Add example applications
103+
if (BUILD_APPLICATIONS)
104+
add_subdirectory(applications)
105+
endif()
106+
102107
generate_aie_ci_test_list("ci_tests.py")

‎LICENSES/CC0-1.0.txt‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
Creative Commons Legal Code
2+
3+
CC0 1.0 Universal
4+
5+
CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
6+
LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
7+
ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
8+
INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
9+
REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
10+
PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
11+
THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
12+
HEREUNDER.
13+
14+
Statement of Purpose
15+
16+
The laws of most jurisdictions throughout the world automatically confer
17+
exclusive Copyright and Related Rights (defined below) upon the creator
18+
and subsequent owner(s) (each and all, an "owner") of an original work of
19+
authorship and/or a database (each, a "Work").
20+
21+
Certain owners wish to permanently relinquish those rights to a Work for
22+
the purpose of contributing to a commons of creative, cultural and
23+
scientific works ("Commons") that the public can reliably and without fear
24+
of later claims of infringement build upon, modify, incorporate in other
25+
works, reuse and redistribute as freely as possible in any form whatsoever
26+
and for any purposes, including without limitation commercial purposes.
27+
These owners may contribute to the Commons to promote the ideal of a free
28+
culture and the further production of creative, cultural and scientific
29+
works, or to gain reputation or greater distribution for their Work in
30+
part through the use and efforts of others.
31+
32+
For these and/or other purposes and motivations, and without any
33+
expectation of additional consideration or compensation, the person
34+
associating CC0 with a Work (the "Affirmer"), to the extent that he or she
35+
is an owner of Copyright and Related Rights in the Work, voluntarily
36+
elects to apply CC0 to the Work and publicly distribute the Work under its
37+
terms, with knowledge of his or her Copyright and Related Rights in the
38+
Work and the meaning and intended legal effect of CC0 on those rights.
39+
40+
1. Copyright and Related Rights. A Work made available under CC0 may be
41+
protected by copyright and related or neighboring rights ("Copyright and
42+
Related Rights"). Copyright and Related Rights include, but are not
43+
limited to, the following:
44+
45+
i. the right to reproduce, adapt, distribute, perform, display,
46+
communicate, and translate a Work;
47+
ii. moral rights retained by the original author(s) and/or performer(s);
48+
iii. publicity and privacy rights pertaining to a person's image or
49+
likeness depicted in a Work;
50+
iv. rights protecting against unfair competition in regards to a Work,
51+
subject to the limitations in paragraph 4(a), below;
52+
v. rights protecting the extraction, dissemination, use and reuse of data
53+
in a Work;
54+
vi. database rights (such as those arising under Directive 96/9/EC of the
55+
European Parliament and of the Council of 11 March 1996 on the legal
56+
protection of databases, and under any national implementation
57+
thereof, including any amended or successor version of such
58+
directive); and
59+
vii. other similar, equivalent or corresponding rights throughout the
60+
world based on applicable law or treaty, and any national
61+
implementations thereof.
62+
63+
2. Waiver. To the greatest extent permitted by, but not in contravention
64+
of, applicable law, Affirmer hereby overtly, fully, permanently,
65+
irrevocably and unconditionally waives, abandons, and surrenders all of
66+
Affirmer's Copyright and Related Rights and associated claims and causes
67+
of action, whether now known or unknown (including existing as well as
68+
future claims and causes of action), in the Work (i) in all territories
69+
worldwide, (ii) for the maximum duration provided by applicable law or
70+
treaty (including future time extensions), (iii) in any current or future
71+
medium and for any number of copies, and (iv) for any purpose whatsoever,
72+
including without limitation commercial, advertising or promotional
73+
purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
74+
member of the public at large and to the detriment of Affirmer's heirs and
75+
successors, fully intending that such Waiver shall not be subject to
76+
revocation, rescission, cancellation, termination, or any other legal or
77+
equitable action to disrupt the quiet enjoyment of the Work by the public
78+
as contemplated by Affirmer's express Statement of Purpose.
79+
80+
3. Public License Fallback. Should any part of the Waiver for any reason
81+
be judged legally invalid or ineffective under applicable law, then the
82+
Waiver shall be preserved to the maximum extent permitted taking into
83+
account Affirmer's express Statement of Purpose. In addition, to the
84+
extent the Waiver is so judged Affirmer hereby grants to each affected
85+
person a royalty-free, non transferable, non sublicensable, non exclusive,
86+
irrevocable and unconditional license to exercise Affirmer's Copyright and
87+
Related Rights in the Work (i) in all territories worldwide, (ii) for the
88+
maximum duration provided by applicable law or treaty (including future
89+
time extensions), (iii) in any current or future medium and for any number
90+
of copies, and (iv) for any purpose whatsoever, including without
91+
limitation commercial, advertising or promotional purposes (the
92+
"License"). The License shall be deemed effective as of the date CC0 was
93+
applied by Affirmer to the Work. Should any part of the License for any
94+
reason be judged legally invalid or ineffective under applicable law, such
95+
partial invalidity or ineffectiveness shall not invalidate the remainder
96+
of the License, and in such case Affirmer hereby affirms that he or she
97+
will not (i) exercise any of his or her remaining Copyright and Related
98+
Rights in the Work or (ii) assert any associated claims and causes of
99+
action with respect to the Work, in either case contrary to Affirmer's
100+
express Statement of Purpose.
101+
102+
4. Limitations and Disclaimers.
103+
104+
a. No trademark or patent rights held by Affirmer are waived, abandoned,
105+
surrendered, licensed or otherwise affected by this document.
106+
b. Affirmer offers the Work as-is and makes no representations or
107+
warranties of any kind concerning the Work, express, implied,
108+
statutory or otherwise, including without limitation warranties of
109+
title, merchantability, fitness for a particular purpose, non
110+
infringement, or the absence of latent or other defects, accuracy, or
111+
the present or absence of errors, whether or not discoverable, all to
112+
the greatest extent permissible under applicable law.
113+
c. Affirmer disclaims responsibility for clearing rights of other persons
114+
that may apply to the Work or any use thereof, including without
115+
limitation any person's Copyright and Related Rights in the Work.
116+
Further, Affirmer disclaims responsibility for obtaining any necessary
117+
consents, permissions or other rights required for any use of the
118+
Work.
119+
d. Affirmer understands and acknowledges that Creative Commons is not a
120+
party to this document and has no duty or obligation with respect to
121+
this CC0 or use of the Work.

‎REUSE.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
version = 1
2+
SPDX-PackageName = "iron"
3+
SPDX-PackageSupplier = "Advanced Micro Devices, Inc."
4+
SPDX-PackageDownloadLocation = "https://github.com/AARInternal/ironclad"
5+
6+
[[annotations]]
7+
path = "applications/llama_3.2_1b/prompt.txt"
8+
precedence = "closest"
9+
SPDX-FileCopyrightText = "Public Domain"
10+
SPDX-License-Identifier = "CC0-1.0"

‎applications/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
add_subdirectory(llama_3.2_1b)

‎applications/llama_3.2_1b/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
add_aie_ci_test(llama_3.2_1b
5+
RUN
6+
"python3 ${CMAKE_CURRENT_SOURCE_DIR}/inference.py /srv/llama3.2-1b/model.safetensors /srv/llama3.2-1b/tokenizer.model --prompt_len 2048 --num_tokens 40"
7+
CHECK
8+
"PASS!"
9+
METRICS
10+
"Total" [=[ Total time: (?P<metric>[\d\.e\+-]+) seconds]=]
11+
"TTFT" [=[ Prefill time: (?P<metric>[\d\.e\+-]+) seconds]=]
12+
"TPS" [=[ Tokens per second: (?P<metric>[\d\.e\+-]+)]=]
13+
"Num Tokens" [=[ Tokens generated: (?P<metric>[\d\.e\+-]+)]=]
14+
)

‎applications/llama_3.2_1b/README.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<!--
2+
SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
3+
SPDX-License-Identifier: Apache-2.0
4+
-->
5+
6+
# Golden Model Inference
7+
8+
## Model and Tokenizer Download Instructions
9+
10+
To download the necessary files for the model, please follow the links below:
11+
12+
- **Model File**: [model.safetensors](https://huggingface.co/meta-llama/Llama-3.2-1B/tree/main)
13+
- **Tokenizer File**: [tokenizer.model](https://huggingface.co/meta-llama/Llama-3.2-1B/tree/main/original)
14+
15+
The CI tests expect these files to be placed in the directory `/srv/llama3.2-1b`;
16+
when calling `inference.py`, you will supply the paths to those two files on the command line, so you can place them anywhere.
17+
18+
## Installation Instructions
19+
20+
Before running `inference.py`, ensure you have the proper environment. To build the environment from scratch, follow the instructions below:
21+
22+
1. Follow the IRON installation instructions in the repository root fist.
23+
After this, you should have an `ironenv` environment set up and activated.
24+
25+
2. Install the following additional requirements:
26+
```
27+
python3 -m pip install -r requirements_examples.txt
28+
```
29+
30+
## Running Inference
31+
32+
Inference with Llama-3.2-1B can be run by specifying a number of tokens to generate based on a prompt. This is done with `inference.py`:
33+
```bash
34+
cd golden_model
35+
python inference.py /path/to/model.safetensors /path/to/tokenizer.model --num_tokens <NUM_TOKENS> --prompt <PROMPT>
36+
```
37+
38+
`inference.py` has the following command format:
39+
```bash
40+
python inference.py <weights_file_path> <tokenizer_file_path> [--num_tokens NUM_TOKENS] [--prompt PROMPT] [--use_prompt_template] [--save_outputs]
41+
```
42+
43+
### Arguments:
44+
- `weights_file_path`: Path to the weights file (e.g., `model.safetensors`).
45+
- `tokenizer_file_path`: Path to the tokenizer file (e.g., `tokenizer.model`).
46+
- `--num_tokens`: (Optional) Number of tokens to predict. Default is `1`.
47+
- `--prompt`: (Optional) Prompt for the model to generate text from. Default is the text in `prompts.txt`.
48+
- `--use_prompt_template`: (Optional) Use a prompt template for the model. Should be passed in when using Instruct weights.
49+
- `--save_outputs`: (Optional) Enable hooks to save outputs of at each layer of the model.

‎applications/llama_3.2_1b/configs/llama32_1b.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"model_config": {
3+
"vocab_size": 128256,
4+
"context_length": 131072,
5+
"emb_dim": 2048,
6+
"n_heads": 32,
7+
"n_layers": 16,
8+
"hidden_dim": 8192,
9+
"n_kv_groups": 8,
10+
"use_kv_cache": true,
11+
"use_aie_gemv": true,
12+
"rope_base": 500000.0,
13+
"dtype": "bfloat16",
14+
"use_aie_final_norm": true,
15+
"use_aie_ffn_gemm": true,
16+
"use_aie_ffn_silu": true,
17+
"use_aie_ffn_mul": true,
18+
"use_aie_attn_projection_gemm": true,
19+
"use_aie_rope": true,
20+
"use_aie_norm1": true,
21+
"use_aie_norm2": true,
22+
"use_aie_residual": true,
23+
"use_aie_regular_mha": false,
24+
"use_aie_fused_mha": true,
25+
"use_aie_final_gemm": false,
26+
"rope_freq": {
27+
"factor": 32.0,
28+
"low_freq_factor": 1.0,
29+
"high_freq_factor": 4.0,
30+
"original_context_length": 8192
31+
}
32+
},
33+
"aie_config": {
34+
"device": "npu2"
35+
}
36+
}

‎applications/llama_3.2_1b/configs/llama32_1b.json.license‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Copyright (c) Sebastian Raschka under Apache License 2.0.
2+
Source for "Build a Large Language Model From Scratch"
3+
- https://www.manning.com/books/build-a-large-language-model-from-scratch
4+
Code: https://github.com/rasbt/LLMs-from-scratch/blob/main/ch05/07_gpt_to_llama/standalone-llama32.ipynb
5+
6+
SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
7+
SPDX-License-Identifier: Apache-2.0

0 commit comments

Comments
 (0)