From 7f9851ae288cc33ae47d481bc09822946daa9c8f Mon Sep 17 00:00:00 2001 From: Kevin Nadro Date: Tue, 1 Jul 2025 13:01:42 -0500 Subject: [PATCH] [Chore]: Added url-checker, updated circular-deps, documented new static analysis .txt pattern (#7442) * fix: ignoring url checker files * fix: url checker * fix: auto fmt and cleanup * fix: moving the bash scripts and known files into the scripts repo * fix: removed all url_results and made it be all in memory * fix: fixed the newline issue * fix: url checking as a step to the static analysis * fix: removed old code * chore: writing documentation on our static checker pattern * fix: updating the docs more to be clearer * fix: copy and paste without understanding requirements of ci cd dependencies? do i need all of these? * fix: updating * fix: I thought this got in? * Update CONTRIBUTING.md Co-authored-by: Jace Browning --------- Co-authored-by: Jace Browning --- .github/workflows/static-analysis.yml | 30 ++++++++++ .gitignore | 2 +- CONTRIBUTING.md | 41 ++++++++++++++ package.json | 5 +- scripts/diff-circular-deps.sh | 2 +- scripts/diff-url-checker.sh | 5 ++ .../known/circular.txt | 0 scripts/known/urls.txt | 21 +++++++ scripts/url-checker.sh | 56 +++++++++++++++++++ 9 files changed, 159 insertions(+), 3 deletions(-) create mode 100755 scripts/diff-url-checker.sh rename known-circular.txt => scripts/known/circular.txt (100%) create mode 100644 scripts/known/urls.txt create mode 100755 scripts/url-checker.sh diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 12805fd5d..bc07de134 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -120,6 +120,36 @@ jobs: - run: npm run circular-deps:diff + npm-url-checker: + runs-on: ubuntu-latest + needs: npm-build-wasm + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + - run: npm install + + - name: Download all artifacts + uses: actions/download-artifact@v4 + + - name: Copy prepared wasm + run: | + ls -R prepared-wasm + cp prepared-wasm/kcl_wasm_lib_bg.wasm public + mkdir rust/kcl-wasm-lib/pkg + cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg + + - name: Copy prepared ts-rs bindings + run: | + ls -R prepared-ts-rs-bindings + mkdir rust/kcl-lib/bindings + cp -r prepared-ts-rs-bindings/* rust/kcl-lib/bindings/ + + - run: npm run url-checker:diff + python-codespell: runs-on: ubuntu-22.04 steps: diff --git a/.gitignore b/.gitignore index 25e5edc45..562e471c2 100644 --- a/.gitignore +++ b/.gitignore @@ -87,4 +87,4 @@ venv .vscode-test .biome/ -.million +.million \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6427419d3..d124a1bf1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -235,6 +235,47 @@ To display logging (to the terminal or console) set `ZOO_LOG=1`. This will log s To enable memory metrics, build with `--features dhat-heap`. +## Running scripts + +There are multiple scripts under the folder path `./scripts` which can be used in various settings. + +### Pattern for a static file, npm run commands, and CI-CD checks + +If you want to implement a static checker follow this pattern. Two static checkers we have are circular dependency checks in our typescript code and url checker to see if any hard coded URL is the typescript application 404s. We have a set of known files in `./scripts/known/*.txt` which is the baseline. + +If you improve the baseline, run the overwrite command and commit the new smaller baseline. Try not to make the baseline bigger, the CI CD will complain. +These baselines are to hold us to higher standards and help implement automated testing against the repository + +#### Output result to stdout +- `npm run circular-deps` +- `npm run url-checker` + +- create a `.sh` file that will run the static checker then output the result to `stdout` + +#### Overwrite result to known .txt file on disk + +If the application needs to overwrite the known file on disk use this pattern. This known .txt file will be source controlled as the baseline + +- `npm run circular-deps:overwrite` +- `npm run url-checker:overwrite` + +#### Diff baseline and current + +These commands will write a /tmp/ file on disk and compare it to the known file in the repository. This command will also be used in the CI CD pipeline for automated checks + +- create a `diff-.sh` file that is the script to diff your tmp file to the baseline +e.g. `diff-url-checker.sh` +```bash +#!/bin/bash +set -euo pipefail + +npm run url-checker > /tmp/urls.txt +diff --ignore-blank-lines -w /tmp/urls.txt ./scripts/known/urls.txt +``` + +- `npm run circular-deps:diff` +- `npm run url-checker:diff` + ## Proposing changes Before you submit a contribution PR to this repo, please ensure that: diff --git a/package.json b/package.json index 5d56502a2..07ebc3da8 100644 --- a/package.json +++ b/package.json @@ -110,8 +110,11 @@ "remove-importmeta": "sed -i 's/import.meta.url/window.location.origin/g' \"./rust/kcl-wasm-lib/pkg/kcl_wasm_lib.js\"; sed -i '' 's/import.meta.url/window.location.origin/g' \"./rust/kcl-wasm-lib/pkg/kcl_wasm_lib.js\" || echo \"sed for both mac and linux\"", "lint-fix": "eslint --fix --ext .ts --ext .tsx src e2e packages/codemirror-lsp-client/src rust/kcl-language-server/client/src", "lint": "eslint --max-warnings 0 --ext .ts --ext .tsx src e2e packages/codemirror-lsp-client/src rust/kcl-language-server/client/src", + "url-checker":"./scripts/url-checker.sh", + "url-checker:overwrite":"npm run url-checker > scripts/known/urls.txt", + "url-checker:diff":"./scripts/diff-url-checker.sh", "circular-deps": "dpdm --no-warning --no-tree -T --skip-dynamic-imports=circular src/index.tsx", - "circular-deps:overwrite": "npm run circular-deps | sed '$d' | grep -v '^npm run' > known-circular.txt", + "circular-deps:overwrite": "npm run circular-deps | sed '$d' | grep -v '^npm run' > scripts/known/circular.txt", "circular-deps:diff": "./scripts/diff-circular-deps.sh", "circular-deps:diff:nodejs": "npm run circular-deps:diff || node ./scripts/diff.js", "files:set-version": "echo \"$(jq --arg v \"$VERSION\" '.version=$v' package.json --indent 2)\" > package.json", diff --git a/scripts/diff-circular-deps.sh b/scripts/diff-circular-deps.sh index 425b363b2..9d95765e6 100755 --- a/scripts/diff-circular-deps.sh +++ b/scripts/diff-circular-deps.sh @@ -2,4 +2,4 @@ set -euo pipefail npm run circular-deps | sed '$d' > /tmp/circular-deps.txt -diff --ignore-blank-lines -w /tmp/circular-deps.txt ./known-circular.txt +diff --ignore-blank-lines -w /tmp/circular-deps.txt ./scripts/known/circular.txt diff --git a/scripts/diff-url-checker.sh b/scripts/diff-url-checker.sh new file mode 100755 index 000000000..21b4e4fa2 --- /dev/null +++ b/scripts/diff-url-checker.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +npm run url-checker > /tmp/urls.txt +diff --ignore-blank-lines -w /tmp/urls.txt ./scripts/known/urls.txt diff --git a/known-circular.txt b/scripts/known/circular.txt similarity index 100% rename from known-circular.txt rename to scripts/known/circular.txt diff --git a/scripts/known/urls.txt b/scripts/known/urls.txt new file mode 100644 index 000000000..2b08be00e --- /dev/null +++ b/scripts/known/urls.txt @@ -0,0 +1,21 @@ + +> zoo-modeling-app@0.0.0 url-checker +> ./scripts/url-checker.sh + +URL STATUS +000 https://${BASE_URL} +301 https://discord.gg/JQEpHR7Nt2 +404 https://github.com/KittyCAD/engine/issues/3528 +404 https://github.com/KittyCAD/modeling-app/commit/${ref} +302 https://github.com/KittyCAD/modeling-app/issues/new/choose +302 https://github.com/KittyCAD/modeling-app/issues/new?template=bug_report.yml +302 https://github.com/KittyCAD/modeling-app/issues/new?title=${title}&body=${body} +404 https://github.com/KittyCAD/modeling-app/releases/tag/v${version} +521 https://placekitten.com/200/200 +302 https://reactrouter.com/en/6.16.0/routers/picking-a-router#using-v64-data-apis +302 https://stackoverflow.com/a/57390160/22753272 +302 https://stackoverflow.com/a/58436959/22753272 +303 https://text-to-cad.zoo.dev/dashboard +307 https://zoo.dev/ +308 https://zoo.dev/docs/api/ml/generate-a-cad-model-from-text +308 https://zoo.dev/docs/kcl diff --git a/scripts/url-checker.sh b/scripts/url-checker.sh new file mode 100755 index 000000000..450e86214 --- /dev/null +++ b/scripts/url-checker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -euo pipefail +trap 'echo "$BASH_COMMAND"' ERR + +remove_after_space () { + sed 's/ .*//' +} + +remove_after_backtick () { + sed 's/`.*//' +} + +remove_after_end_paren () { + sed 's/).*//' +} + +remove_after_double_quote () { + sed 's/".*//' +} + +remove_after_gt () { + sed 's/>.*//' +} + +remove_after_comma () { + sed 's/,.*//' +} + +# Search all src/**/*.ts files +val1=$(grep -Eoh "(https)://[^']+" src/**/*.ts | remove_after_space | remove_after_backtick | remove_after_end_paren | remove_after_double_quote | remove_after_gt | remove_after_comma) + +# Search all src/**/*.tsx files +val2=$(grep -Eoh "(https)://[^']+" src/**/*.tsx | remove_after_space | remove_after_backtick | remove_after_end_paren | remove_after_double_quote | remove_after_gt | remove_after_comma) + +# Required a newline between them when combining since there is not one at the end of val1 +combined="$val1"$'\n'"$val2" + +# Merge both ts and tsx results and unique them +uniqued=$(echo "$combined" | sort | uniq) + +# All urls and status codes +all="URL\tSTATUS\n" + +# All non 200 urls and status codes +problematic="URL\tSTATUS\n" +while read line; do + # || true this curl request to bypass any failures and not have the scrip panic. + # the set -euo pipefail will cause a panic if a curl fails + status=$(curl -o /dev/null -s -w "%{http_code}\n" $line || true) + all+="$status\t$line\n" + if [[ "$status" -ne 200 ]]; then + # list status first over line because of white space formatting, less annoying for diffing + problematic+="$status\t$line\n" + fi +done < <(echo "$uniqued") +echo -e $problematic | column -t