Skip to content

Update pack generation #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 15, 2022
Merged

Update pack generation #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c1105d9
Remove anonymized pack generation step
rvermeulen Jul 14, 2022
a0d578a
Remove LGTM pack generation
rvermeulen Jul 14, 2022
4b7e428
Add changenote to describe change to release artifacts.
rvermeulen Jul 14, 2022
947ea95
Merge branch 'main' into rvermeulen/update-pack-generation
rvermeulen Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 0 additions & 189 deletions .github/workflows/code-scanning-pack-gen.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,130 +33,6 @@ jobs:
jq --compact-output '.supported_environment | {include: .}' supported_codeql_configs.json
)"

create-code-scanning-pack-anon:
name: Create anonymous Code Scanning pack
needs: prepare-code-scanning-pack-matrix
runs-on: ubuntu-20.04-xl
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.prepare-code-scanning-pack-matrix.outputs.matrix) }}
steps:
- uses: actions/checkout@v2

- name: Cache CodeQL
id: cache-codeql
uses: actions/cache@v2.1.3
with:
path: ${{ github.workspace }}/codeql_home
key: codeql-home-${{ matrix.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}

- name: Install CodeQL
if: steps.cache-codeql.outputs.cache-hit != 'true'
uses: ./.github/actions/install-codeql
with:
codeql-cli-version: ${{ matrix.codeql_cli }}
codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
codeql-home: ${{ github.workspace }}/codeql_home
add-to-path: false

- name: Install Python
uses: actions/setup-python@v4
with:
python-version: "3.9"

- name: Anonymising and pre-compiling queries
env:
CODEQL_HOME: ${{ github.workspace }}/codeql_home
run: |
PATH=$PATH:$CODEQL_HOME/codeql
pip install -r scripts/requirements.txt
find rule_packages/cpp -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py -a cpp
find rule_packages/c -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py --skip-shared-test-generation -a c

echo "Remove help files that cannot be freely distributed"
find cpp/autosar/src/rules -name "*.md" -delete
find c/misra/src/rules -name "*.md" -delete

codeql query compile --search-path cpp --threads 0 cpp
codeql query compile --search-path c --search-path cpp --threads 0 c
cd ..
zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports

- name: Upload GHAS Query Pack
uses: actions/upload-artifact@v2
with:
name: code-scanning-cpp-query-pack-anon.zip
path: code-scanning-cpp-query-pack-anon.zip

- name: Create LGTM query pack
env:
CODEQL_HOME: ${{ github.workspace }}/codeql_home
run: |
PATH=$PATH:$CODEQL_HOME/codeql
mkdir -p lgtm-cpp-query-pack
function copy_queries_for_pack {
for q in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls)
do
copy_from_root_name="${q%.*}"
copy_to_root_name=$(realpath --relative-to "./$2/$1/src/" "$copy_from_root_name")
query_dir=$(dirname "lgtm-cpp-query-pack/$copy_to_root_name")
mkdir -p "$query_dir"
# Copy each selected ql file
cp "$copy_from_root_name.ql" "lgtm-cpp-query-pack/$copy_to_root_name.ql"
done
}

echo "Copying autosar-default queries (CPP)"
copy_queries_for_pack "autosar" "cpp"
echo "Copying cert-default queries (CPP)"
copy_queries_for_pack "cert" "cpp"

echo "Copying misra-default queries (C)"
copy_queries_for_pack "misra" "c"
echo "Copying cert-default queries (C)"
copy_queries_for_pack "cert" "c"


# Now copy all the .qll files
lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
for query_pack in autosar cert common
do
echo "Copying $query_pack qll files"
pushd cpp/$query_pack/src
for query_library in $(find . -name \*.qll)
do
qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
echo "Making $qll_dir"
mkdir -p "$qll_dir"
cp "$query_library" "$lgtm_pack_dir/$query_library"
done
popd
done


for query_pack in misra cert common
do
echo "Copying $query_pack qll files"
pushd c/$query_pack/src
for query_library in $(find . -name \*.qll)
do
qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
echo "Making $qll_dir"
mkdir -p "$qll_dir"
cp "$query_library" "$lgtm_pack_dir/$query_library"
done
popd
done

cd lgtm-cpp-query-pack
zip -9 -r ../lgtm-cpp-query-pack-anon.zip *

- name: Upload LGTM query pack
uses: actions/upload-artifact@v2
with:
name: lgtm-cpp-query-pack-anon.zip
path: lgtm-cpp-query-pack-anon.zip

create-code-scanning-pack:
name: Create Code Scanning pack
needs: prepare-code-scanning-pack-matrix
Expand Down Expand Up @@ -217,68 +93,3 @@ jobs:
with:
name: code-scanning-cpp-query-pack.zip
path: code-scanning-cpp-query-pack.zip

- name: Create LGTM query pack
env:
CODEQL_HOME: ${{ github.workspace }}/codeql_home
run: |
PATH=$PATH:$CODEQL_HOME/codeql
mkdir -p lgtm-cpp-query-pack
function copy_queries_for_pack {
for rule_dir in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
do
copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./$2/$1/src/" "$rule_dir")"
mkdir -p "$copy_to_root"
# Copy each selected ql file, plus the related files
find "$rule_dir" -name '*.ql' -o -name '*.c' -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
done
}
echo "Copying autosar-default queries (CPP)"
copy_queries_for_pack "autosar" "cpp"
echo "Copying cert-default queries (CPP)"
copy_queries_for_pack "cert" "cpp"

echo "Copying misra-default queries (C)"
copy_queries_for_pack "misra" "c"
echo "Copying cert-default queries (C)"
copy_queries_for_pack "cert" "c"


# Now copy all the .qll files
lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
for query_pack in autosar cert common
do
echo "Copying $query_pack qll files"
pushd cpp/$query_pack/src
for query_library in $(find . -name \*.qll)
do
qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
echo "Making $qll_dir"
mkdir -p "$qll_dir"
cp "$query_library" "$lgtm_pack_dir/$query_library"
done
popd
done

for query_pack in misra cert common
do
echo "Copying $query_pack qll files"
pushd c/$query_pack/src
for query_library in $(find . -name \*.qll)
do
qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
echo "Making $qll_dir"
mkdir -p "$qll_dir"
cp "$query_library" "$lgtm_pack_dir/$query_library"
done
popd
done

cd lgtm-cpp-query-pack
zip -9 -r ../lgtm-cpp-query-pack.zip *

- name: Upload LGTM query pack
uses: actions/upload-artifact@v2
with:
name: lgtm-cpp-query-pack.zip
path: lgtm-cpp-query-pack.zip
2 changes: 2 additions & 0 deletions change_notes/2022-07-14-update-pack-generation.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- Remove support for the anonymized pack because it is no longer necessary. The code scanning pack is superseding this pack.
- Remove support for the LGTM pack because LGTM is EOL.