diff --git a/.github/scripts/dry_run.sh b/.github/scripts/dry_run.sh new file mode 100755 index 0000000..a2243b2 --- /dev/null +++ b/.github/scripts/dry_run.sh @@ -0,0 +1,31 @@ +set -e + +echo "*** Updating cabal ***" + +cabal update + +echo "*** Installing clc-stackage ***" + +# --overwrite-policy=always and deleting output/ are unnecessary for CI since +# this script will only be run one time, but it's helpful when we are +# testing the script locally. +cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always + +if [[ -d output ]]; then + rm -r output +fi + +echo "*** Building all with --dry-run ***" + +set +e +./bin/clc-stackage --batch 100 --cabal-options="--dry-run" + +ec=$? + +if [[ $ec != 0 ]]; then + echo "*** clc-stackage failed ***" + .github/scripts/print_logs.sh + exit 1 +else + echo "*** clc-stackage succeeded ***" +fi diff --git a/.github/scripts/print_logs.sh b/.github/scripts/print_logs.sh new file mode 100755 index 0000000..26d0648 --- /dev/null +++ b/.github/scripts/print_logs.sh @@ -0,0 +1,12 @@ +if [[ ! -d output/logs ]]; then + echo "*** No output ***" +else + cd output/logs + + for dir in */; do + echo "*** $dir stdout ***" + cat "$dir/stdout.log" + echo "*** $dir stderr ***" + cat "$dir/stderr.log" + done +fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff7718a..1de8319 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,13 +29,15 @@ jobs: - uses: actions/checkout@v4 - uses: haskell-actions/setup@v2 with: - ghc-version: "9.10.2" + # Should be the current stackage nightly, though this will likely go + # out-of-date eventually, until a problem is reported. + ghc-version: "9.12" - name: Configure run: | cabal configure --enable-tests --ghc-options -Werror - name: Build executable - run: cabal build clc-stackage + run: cabal build exe:clc-stackage - name: Unit Tests id: unit @@ -54,17 +56,22 @@ jobs: - name: Print functional failures if: ${{ failure() && steps.functional.conclusion == 'failure' }} shell: bash - run: | + run: .github/scripts/print_logs.sh + nix: + strategy: + fail-fast: false + matrix: + os: + - "ubuntu-latest" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 - if [[ ! -d output/logs ]]; then - echo "*** No output ***" - else - cd output/logs + - name: Setup nix + uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_path: nixpkgs=channel:nixos-unstable - for dir in */; do - echo "*** $d stdout ***" - cat "$dir/stdout.log" - echo "*** $d stderr ***" - cat "$dir/stderr.log" - done - fi + - name: Dry run + run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh' diff --git a/README.md b/README.md index 75a143e..a9c6def 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,13 @@ An impact assessment is due when The procedure is as follows: -1. Rebase changes, mandated by your proposal, atop of `ghc-9.10` branch. +1. Rebase changes, mandated by your proposal, atop the ghc branch (or tag) that corresponds to the current [stackage nightly](https://www.stackage.org/nightly). For example, if the latest snapshot ghc is `ghc-9.12.3`, we would want to rebase our changes on the `ghc-9.12.3-release` tag. 2. Compile a patched GHC, say, `~/ghc/_build/stage1/bin/ghc`. 3. `git clone https://github.com/haskell/clc-stackage`, then `cd clc-stackage`. -4. Build the exe: `cabal install clc-stackage --installdir=./bin`. +4. Build the exe: `cabal install exe:clc-stackage`. > :warning: **Warning:** Use a normal downloaded GHC for this step, **not** your custom built one. Why? Using the custom GHC can force a build of many dependencies you'd otherwise get for free e.g. `vector`. @@ -30,12 +30,13 @@ The procedure is as follows: with-compiler: /home/ghc/_build/stage1/bin/ghc ``` -6. Run `./bin/clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. +6. Run `clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. * On a recent Macbook Air it takes around 12 hours, YMMV. * You can interrupt `cabal` at any time and rerun again later. * Consider setting `--jobs` to retain free CPU cores for other tasks. * Full build requires roughly 7 Gb of free disk space. + * If the build fails with an error about max amount of arguments in `gcc`, run again, but with smaller batch size. 250 worked well for me. To get an idea of the current progress, we can run the following commands on the log file: @@ -59,6 +60,61 @@ The procedure is as follows: 8. When everything finally builds, get back to CLC with a list of packages affected and patches required. +### Troubleshooting + +Because we build with `nightly` and are at the mercy of cabal's constraint solver, it is possible to run into solver / build issues that have nothing to do with our custom GHC. Some of the most common problems include: + +- Nightly adds a new, problematic package `p` e.g. + + - `p` requires a new system dependency (e.g. a C library). + - `p` is an executable. + - `p` depends on a package in [./excluded_pkgs.jsonc](excluded_pkgs.jsonc). + +- A cabal flag is set in a way that breaks the build. For example, our snapshot requires that the `bson` library does *not* have its `_old-network` flag set, as this will cause a build error with our version of `network`. This flag is automatic, so we have to force it in `generated/cabal.project` with `constraints: bson -_old-network`. + +- Nightly has many packages drop out for some reason, increasing the chance for solver non-determinism. + +We attempt to mitigate such issues by: + +- Writing most of the snapshot's exact package versions as cabal constraints to the generated `./generated/cabal.project.local`, which ensures we (transitively) build the same package version every time. Note that boot packages like `text` are deliberately excluded so that we can build a snapshot with multiple GHCs. Otherwise even a GHC minor version difference would fail because `ghc` is in the build plan. + +- Ignoring bounds in `generated/cabal.project`: + + ``` + allow-newer: *:* + allow-older: *:* + ``` + +Nevertheless, it is still possible for issues to slip through. When a package `p` fails to build for some reason, we should first: + +- Verify that `p` is not in `excluded_pkgs.jsonc`. If it is, nightly probably pulled in some new reverse-dependency `q` that should be added to `excluded_pkgs.jsonc`. + +- Verify that `p` does not have cabal flags that can affect dependencies / API. + +- Verify that `p`'s version matches what it is in the current snapshot (e.g. `https://www.stackage.org/nightly`). If it does not, either a package needs to be excluded or constraints need to be added. + +In general, user mitigations for solver / build problems include: + +- Adding `p` to `excluded_pkgs.jsonc`. Note that `p` will still be built if it is a (transitive) dependency of some other package in the snapshot, but will not have its exact bounds written to `cabal.project.local`. + +- Manually downloading a snapshot (e.g. `https://www.stackage.org/nightly/cabal.config`), changing / removing the offending package(s), and supplying the file with the `--snapshot-path` param. Like `excluded_pkgs.jsonc`, take care that the problematic package is not a (transitive) dependency of something in the snapshot. + +- Adding constraints to `generated/cabal.project` e.g. flags or version constraints like `constraints: filepath > 1.5`. + +#### Misc + +- Note that while a GHC minor version difference is usually okay, a GHC *major* difference will very likely lead to errors. + +- The `flake.nix` line: + + ```nix + compiler = pkgs.haskell.packages.ghc; + ``` + + can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything with `--dry-run`, which should report solver errors (e.g. bounds) at the very least. + +- If you encounter an error that you think indicates a problem with the configuration here (e.g. new package needs to be excluded, new constraint added), please open an issue. While that is being resolved, the mitigations from the [previous section](#troubleshooting) may be useful. + ### The clc-stackage exe `clc-stackage` is an executable that will: @@ -72,7 +128,7 @@ The procedure is as follows: By default, `clc-stackage` queries https://www.stackage.org/ for snapshot information. In situations where this is not desirable (e.g. the server is not working, or we want to test a custom snapshot), the snapshot can be overridden: ```sh -$ ./bin/clc-stackage --snapshot-path=path/to/snapshot +$ clc-stackage --snapshot-path=path/to/snapshot ``` This snapshot should be formatted similar to the `cabal.config` endpoint on the stackage server (e.g. https://www.stackage.org/nightly/cabal.config). That is, package lines should be formatted ` ==`: @@ -94,15 +150,23 @@ By default (`--write-logs save-failures`), the build logs are saved to the `./ou #### Group batching -The `clc-stackage` exe allows for splitting the entire package set into subset groups of size `N` with the `--batch N` option. Each group is then built sequentially. Not only can this be useful for situations where building the entire package set in one go is infeasible, but it also provides a "cache" functionality, that allows us to interrupt the program at any point (e.g. `CTRL-C`), and pick up where we left off. For example: +By default, the `clc-stackage` exe tries to build all packages at once i.e. every package is written to `generated/generated.cabal`. This can cause problems e.g. we do not have enough memory to build everything simultaneously, or we receive an error that `gcc` has been given too many arguments. Hence we provide the `--batch N` option, which will split the package set into disjoint groups of size `N`. Each group is then built sequentially. -```sh -$ ./bin/clc-stackage --batch 100 -``` +The default behavior is: + + 1. `clc-stackage` will try to build everything in the same group, even if some package fails (equivalent to cabal's `--keep-going` flag.). If instead `--package-fail-fast` is enabled, the first failure will cause the entire group to immediately fail, and we will move onto the next group. -This will split the entire downloaded package set into groups of size 100. Each time a group finishes (success or failure), stdout/err will be updated, and then the next group will start. If the group failed to build and we have `--write-logs save-failures` (the default), then the logs and error output will be in `./output/logs//`, where `` is the name of the first package in the group. + 2. `clc-stackage` will try every group, even if some prior group fails. The `--group-fail-fast` option changes this so that the first failure will cause `clc-stackage` to exit. + +Each time a group finishes (success or failure), stdout/err will be updated, and then the next group will start. If the group failed to build and we have `--write-logs save-failures` (the default), then the logs and error output will be in `./output/logs//`, where `` is the name of the first package in the group. + +When `clc-stackage` itself finishes (either on its own or via an interrupt like `CTRL-C`), the results are saved to a cache which records all successes, failures, and untested packages. This allows us to pick up where we left off with untested packages (including failures if the `--retry-failures` flag is active). + +> [!IMPORTANT] +> +> The cache operates at the *batch group* level, so only packages that have been part of a successful group will be considered successes. Conversely, a package will be considered a failure if it is part of a failing group, even if it was built successfully. Therefore, to see what packages actually failed, we will want to check the logs directory. Alternatively, we can first run `clc-stackage` initially with a large `--batch` group (for maximum performance), then run it again with, say, `--batch 1`. -See `./bin/clc-stackage --help` for more info. +See `clc-stackage --help` for more info. ##### Optimal performance diff --git a/app/Main.hs b/app/Main.hs index 4f00334..ade9eb6 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -11,6 +11,4 @@ main = do let hLogger = Logging.mkDefaultLogger case mWidth of Just w -> Runner.run $ hLogger {Logging.terminalWidth = w} - Nothing -> do - Logging.putTimeInfoStr hLogger "Failed detecting terminal width" - Runner.run hLogger + Nothing -> Runner.run hLogger diff --git a/cabal.project b/cabal.project index 08f105a..43ae235 100644 --- a/cabal.project +++ b/cabal.project @@ -1,5 +1,3 @@ -index-state: 2025-05-22T04:18:43Z - packages: . program-options @@ -17,6 +15,7 @@ program-options -Wprepositive-qualified-module -Wredundant-constraints -Wunused-binds + -Wunused-packages -Wunused-type-patterns -Wno-unticked-promoted-constructors diff --git a/clc-stackage.cabal b/clc-stackage.cabal index 52662a7..fdfe9c7 100644 --- a/clc-stackage.cabal +++ b/clc-stackage.cabal @@ -26,59 +26,7 @@ common common-lang default-language: GHC2021 -library utils - import: common-lang - exposed-modules: - CLC.Stackage.Utils.Exception - CLC.Stackage.Utils.IO - CLC.Stackage.Utils.JSON - CLC.Stackage.Utils.Logging - CLC.Stackage.Utils.OS - CLC.Stackage.Utils.Package - CLC.Stackage.Utils.Paths - - build-depends: - , aeson >=2.0 && <2.3 - , aeson-pretty ^>=0.8.9 - , base >=4.16.0.0 && <4.22 - , bytestring >=0.10.12.0 && <0.13 - , deepseq >=1.4.6.0 && <1.6 - , directory ^>=1.3.5.0 - , file-io ^>=0.1.0.0 - , filepath >=1.5.0.0 && <1.6 - , os-string ^>=2.0.0 - , pretty-terminal ^>=0.1.0.0 - , text >=1.2.3.2 && <2.2 - , time >=1.9.3 && <1.15 - - hs-source-dirs: src/utils - -library parser - import: common-lang - exposed-modules: - CLC.Stackage.Parser - CLC.Stackage.Parser.API - CLC.Stackage.Parser.API.CabalConfig - CLC.Stackage.Parser.API.Common - CLC.Stackage.Parser.API.JSON - - build-depends: - , aeson - , base - , bytestring - , containers >=0.6.3.1 && <0.9 - , deepseq - , filepath - , http-client >=0.5.9 && <0.8 - , http-client-tls ^>=0.3 - , http-types ^>=0.12.3 - , text - , utils - - hs-source-dirs: src/parser - ghc-options: -Wunused-packages - -library builder +library import: common-lang exposed-modules: CLC.Stackage.Builder @@ -86,55 +34,56 @@ library builder CLC.Stackage.Builder.Env CLC.Stackage.Builder.Process CLC.Stackage.Builder.Writer - - build-depends: - , base - , containers - , directory - , filepath - , process ^>=1.6.9.0 - , text - , utils - - hs-source-dirs: src/builder - ghc-options: -Wunused-packages - -library runner - import: common-lang - exposed-modules: + CLC.Stackage.Parser + CLC.Stackage.Parser.API + CLC.Stackage.Parser.API.CabalConfig + CLC.Stackage.Parser.API.Common + CLC.Stackage.Parser.API.JSON + CLC.Stackage.Parser.Utils CLC.Stackage.Runner CLC.Stackage.Runner.Args CLC.Stackage.Runner.Env CLC.Stackage.Runner.Report + CLC.Stackage.Utils.Exception + CLC.Stackage.Utils.IO + CLC.Stackage.Utils.JSON + CLC.Stackage.Utils.Logging + CLC.Stackage.Utils.OS + CLC.Stackage.Utils.Package + CLC.Stackage.Utils.Paths build-depends: - , aeson - , base - , builder - , containers - , directory - , filepath - , optparse-applicative >=0.16.1.0 && <0.19 - , parser - , pretty-terminal - , text - , time - , utils - - hs-source-dirs: src/runner - ghc-options: -Wunused-packages + , aeson >=2.0 && <2.3 + , aeson-pretty ^>=0.8.9 + , base >=4.16.0.0 && <4.23 + , bytestring >=0.10.12.0 && <0.13 + , containers >=0.6.3.1 && <0.9 + , deepseq >=1.4.6.0 && <1.6 + , directory ^>=1.3.5.0 + , file-io >=0.1.0.0 && <0.3 + , filepath >=1.5.0.0 && <1.6 + , http-client >=0.5.9 && <0.8 + , http-client-tls >=0.3 && <0.5 + , http-types ^>=0.12.3 + , optparse-applicative ^>=0.19.0.0 + , os-string ^>=2.0.0 + , pretty-terminal ^>=0.1.0.0 + , process ^>=1.6.9.0 + , text >=1.2.3.2 && <2.2 + , time >=1.9.3 && <1.16 + + hs-source-dirs: src executable clc-stackage import: common-lang main-is: Main.hs build-depends: , base - , runner + , clc-stackage , terminal-size ^>=0.3.4 - , utils hs-source-dirs: ./app - ghc-options: -threaded -with-rtsopts=-N -Wunused-packages + ghc-options: -threaded -with-rtsopts=-N library test-utils import: common-lang @@ -145,7 +94,6 @@ library test-utils , tasty-golden ^>=2.3.1.1 hs-source-dirs: test/utils - ghc-options: -Wunused-packages test-suite unit import: common-lang @@ -160,22 +108,19 @@ test-suite unit build-depends: , base - , builder + , clc-stackage , containers , deepseq , filepath , http-client-tls - , parser - , runner , tasty , tasty-golden , tasty-hunit >=0.9 && <0.11 , test-utils , time - , utils hs-source-dirs: test/unit - ghc-options: -threaded -with-rtsopts=-N -Wunused-packages + ghc-options: -threaded -with-rtsopts=-N test-suite functional import: common-lang @@ -183,21 +128,15 @@ test-suite functional main-is: Main.hs build-depends: , base - , builder , bytestring + , clc-stackage , containers , env-guard ^>=0.2 , filepath - , runner , tasty , tasty-golden , test-utils , text , time - , utils hs-source-dirs: test/functional - --- For some reason -Wunused-packages is complaining about clc-stackage --- being an unnecessary dep for the functional test suite...hence it is --- removed from cabal.project and added manually to other targets. diff --git a/dev.md b/dev.md index cd87162..186cfbd 100644 --- a/dev.md +++ b/dev.md @@ -8,36 +8,32 @@ This project is organized into several libraries and a single executable. Roughl 2. Prune `s` based on packages we know we do not want (e.g. system deps). 3. Generate a custom `generated.cabal` file for the given package set, and try to build it. -Futhermore, we allow for building subsets of the entire stackage package set with the `--batch` feature. This will split the package set into disjoint groups, and build each group sequentially. The process can be interrupted at any time (e.g. `CTRL-C`), and progress will be saved in a "cache" (json file), so we can pick up where we left off. +Furthermore, we allow for building subsets of the entire stackage package set with the `--batch` feature. This will split the package set into disjoint groups, and build each group sequentially. The process can be interrupted at any time (e.g. `CTRL-C`), and progress will be saved in a "cache" (json file), so we can pick up where we left off. ## Components +The `clc-stackage` library is namespaced by functionality: + ### utils -`utils` is a library containing common utilities e.g. logging and hardcoded file paths. +`CLC.Stackage.Utils` contains common utilities e.g. logging and hardcoded file paths. ### parser -`parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [excluded_pkgs.json](excluded_pkgs.json). The primary function is: +`CLC.Stackage.Parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [excluded_pkgs.json](excluded_pkgs.json). The primary function is: ```haskell -- CLC.Stackage.Parser -getPackageList :: IO [PackageResponse] +getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] ``` -If you want to get the list of the packages to be built (i.e. stackage_snapshot - excluded_packages), load the parser into the repl with `cabal repl parser`, and run the following: - -```haskell --- CLC.Stackage.Parser --- printPackageList :: Bool -> Maybe Os -> IO () -λ. printPackageList True Nothing -``` +If you want to get the list of the packages to be built (i.e. stackage_snapshot - excluded_packages), run `clc-stackage --print-package-set`. This will write the package list used for each OS to `pkgs_.txt`. ### builder -`builder` is responsible for building a given package set. The primary functions are: +`CLC.Stackage.Builder` is responsible for building a given package set. The primary functions are: ```haskell -- CLC.Stackage.Builder @@ -58,7 +54,7 @@ That is: ### runner -`runner` orchestrates everything. The primary function is: +`CLC.Stackage.Runner` orchestrates everything. The primary function is: ```haskell run :: Logging.Handle -> IO () @@ -77,30 +73,29 @@ The reason this logic is a library function and not the executable itself is for The executable that actually runs. This is a very thin wrapper over `runner`, which merely sets up the logging handler. -## Updating to a new shapshot +## Updating to a new snapshot + +`clc-stackage` is based on `nightly` -- which changes automatically -- meaning we do not necessarily have to do anything when a new (minor) snapshot is released. On the other hand, *major* snapshot updates will almost certainly bring in new packages that need to be excluded, so there are some general "update steps" we will want to take: -1. Update to the desired snapshot: +1. Modify [excluded_pkgs.json](excluded_pkgs.json) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages that should be excluded. - ```haskell - -- CLC.Stackage.Parser.API - stackageSnapshot :: String - stackageSnapshot = "nightly-yyyy-mm-dd" - ``` +2. Update `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). -2. Update the `index-state` in [cabal.project](cabal.project) and [generated/cabal.project](generated/cabal.project). +3. Update functional tests as needed i.e. exact package versions in `*golden` and `test/functional/snapshot.txt`. -3. Modify [excluded_pkgs.json](excluded_pkgs.json) as needed. That is, updating the snapshot will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages that should be excluded. +4. Optional: Update nix: -4. Update references to the current ghc e.g. + - Inputs (`nix flake update`). + - GHC: Update the `compiler = pkgs.haskell.packages.ghc;` line. + - Add to the `flake.nix`'s `ldDeps` and `deps` as needed to have the `nix` CI job pass. System libs available on nix can be found here: https://search.nixos.org/packages?channel=unstable. - 1. `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). - 2. [README.md](README.md). + This job builds everything with `--dry-run`, so its success is a useful proxy for `clc-stackage`'s health. In other words, if the nix job fails, there is almost certainly a general issue (i.e. either a package should be excluded or new system dep is required), but if it succeeds, the package set is in pretty good shape (there may still be sporadic issues e.g. a package does not properly declare its system dependencies at config time). -5. Update functional tests as needed i.e. exact package versions in `*golden` and `test/functional/snapshot.txt`. +5. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). -6. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). +### Verifying snapshot -7. Optional: Update nix inputs (`nix flake update`). +To verify the snapshot, every package should actually be built i.e. run `clc-stackage` as you normally would. However, this can be quite time-consuming when there are new solver errors that need to be resolved (e.g. new system deps need to be added). An easier method is to first get everything passing with `dry-run` -- e.g. `clc-stackage --cabal-options="--dry-run"` -- then once that is passing, run `clc-stackage` for real. ## Testing diff --git a/excluded_pkgs.json b/excluded_pkgs.jsonc similarity index 71% rename from excluded_pkgs.json rename to excluded_pkgs.jsonc index 581b48e..7c58774 100644 --- a/excluded_pkgs.json +++ b/excluded_pkgs.jsonc @@ -12,10 +12,22 @@ "ALUT", "amqp-utils", "arbtt", + "array", // see NOTE: [Boot packages] + // NOTE: [Boot packages] + // + // Boot packages are excluded from directly building here -- and having + // their constraints written -- because + // their versions are not stable within the same major ghc version, + // and a version mismatch will cause a build failure whenever ghc + // is in the build plan (which is true for stackage). + // + // Packages taken from: + // https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history "base", "beam-postgres", "bench", "bhoogle", + "binary", // see NOTE: [Boot packages] "bindings-libzip", "blas-carray", "blas-comfort-array", @@ -26,13 +38,16 @@ "bugsnag", "bugsnag-wai", "bugsnag-yesod", + "bytestring", // see NOTE: [Boot packages] "c2hs", + "Cabal", // see NOTE: [Boot packages] "cabal-clean", "cabal-flatpak", "cabal-install", "cabal-install-solver", "cabal-rpm", "cabal-sort", + "Cabal-syntax", // see NOTE: [Boot packages] "cabal2nix", "calendar-recycling", "Clipboard", @@ -40,6 +55,7 @@ "comfort-blas", "comfort-fftw", "comfort-glpk", + "containers", // see NOTE: [Boot packages] "core-telemetry", "countdown-numbers-game", "cql-io", @@ -48,7 +64,10 @@ "cuda", "cutter", "dbcleaner", + "dbus-menu", // gtk-3 + "deepseq", // see NOTE: [Boot packages] "diagrams-svg", + "directory", // see NOTE: [Boot packages] "discount", "dl-fedora", "doctest-extract", @@ -58,13 +77,16 @@ "emd", "equal-files", "essence-of-live-coding-pulse", + "exceptions", // see NOTE: [Boot packages] "experimenter", "fbrnch", "fedora-haskell-tools", "fedora-repoquery", "fft", "fftw-ffi", + "file-io", // see NOTE: [Boot packages] "file-modules", + "filepath", // see NOTE: [Boot packages] "fix-whitespace", "flac", "flac-picture", @@ -75,12 +97,20 @@ "fsnotify-conduit", "gauge", "gd", - "ghc", - "ghc-bignum", + "ghc", // see NOTE: [Boot packages] + "ghc-bignum", // see NOTE: [Boot packages] + "ghc-boot", // see NOTE: [Boot packages] + "ghc-boot-th", // see NOTE: [Boot packages] + "ghc-compact", // see NOTE: [Boot packages] "ghc-core", - "ghc-internal", + "ghc-experimental", // see NOTE: [Boot packages] + "ghc-heap", // see NOTE: [Boot packages] + "ghc-internal", // see NOTE: [Boot packages] + "ghc-platform", // see NOTE: [Boot packages] "ghc-prim", "ghc-syntax-highlighter", + "ghc-toolchain", // see NOTE: [Boot packages] + "ghci", // see NOTE: [Boot packages] "ghostscript-parallel", "gi-atk", "gi-cairo", @@ -106,6 +136,7 @@ "gi-gtk3", "gi-gtk4", "gi-gtk-hs", + "gi-gtk-layer-shell", "gi-gtksource", "gi-gtksource5", "gi-harfbuzz", @@ -135,14 +166,18 @@ "gtk3", "H", "hackage-cli", + "haddock-api", // see NOTE: [Boot packages] + "haddock-library", // see NOTE: [Boot packages] "hamtsolo", "happstack-server-tls", "happy", + "haskeline", // see NOTE: [Boot packages] "haskell-gi", "haskell-gi-base", "haskell-gi-overloading", "haskoin-core", "haskoin-node", + "haskoin-store", "haskoin-store-data", "hasql", "hasql-dynamic-statements", @@ -157,6 +192,7 @@ "hasql-th", "hasql-transaction", "haxr", + "headroom", // segfault "hinotify", "hkgr", "hledger-interest", @@ -167,6 +203,7 @@ "hmatrix-special", "hmm-lapack", "hmpfr", + "hpc", // see NOTE: [Boot packages] "hopenssl", "hp2pretty", "hpqtypes", @@ -195,7 +232,7 @@ "ihs", "Imlib", "inline-r", - "integer-gmp", + "integer-gmp", // see NOTE: [Boot packages] "integer-simple", "ip6addr", "ipython-kernel", @@ -228,16 +265,21 @@ "magico", "mbox-utility", "mega-sdist", + "microformats2-parser", // segfault "midi-alsa", "midi-music-box", "misfortune", "mmark-cli", + "mmark-ext", // ghc-syntax-highlighter "moffy-samples-gtk3", "moffy-samples-gtk3-run", + "moffy-samples-gtk4", + "moffy-samples-gtk4-run", "mpi-hs", "mpi-hs-binary", "mpi-hs-cereal", "mstate", + "mtl", // see NOTE: [Boot packages] "mysql", "mysql-json-table", "mysql-simple", @@ -253,8 +295,10 @@ "opaleye", "openssl-streams", "OrderedBits", + "os-string", // see NOTE: [Boot packages] "pagure-cli", "pandoc-cli", + "parsec", // see NOTE: [Boot packages] "parser-combinators-tests", "pcre-heavy", "pcre-light", @@ -272,7 +316,9 @@ "postgresql-schema", "postgresql-simple", "postgresql-simple-url", + "pretty", // see NOTE: [Boot packages] "primecount", + "process", // see NOTE: [Boot packages] "profiterole", "proto-lens-protobuf-types", "psql-helpers", @@ -292,6 +338,7 @@ "rocksdb-haskell", "rocksdb-haskell-jprupp", "rocksdb-query", + "rts", // see NOTE: [Boot packages] "scrypt", "sdl2", "sdl2-gfx", @@ -299,6 +346,7 @@ "sdl2-mixer", "sdl2-ttf", "secp256k1-haskell", + "semaphore-compat", // see NOTE: [Boot packages] "seqalign", "servant-http-streams", "servius", @@ -312,9 +360,11 @@ "sqlcli", "sqlcli-odbc", "sqlite-simple", + "stakhanov", // hasql "stack-all", "stack-clean-old", "stack-templatizer", + "stm", // see NOTE: [Boot packages] "stringprep", "SVGFonts", "swizzle", @@ -323,28 +373,37 @@ "swizzle-set", "sydtest-persistent-postgresql", "synthesizer-alsa", + "taffybar", "tasty-papi", - "template-haskell", + "template-haskell", // see NOTE: [Boot packages] "termonad", + "terminfo", // see NOTE: [Boot packages] "test-certs", + "text", // see NOTE: [Boot packages] "text-icu", "text-regex-replace", + "time", // see NOTE: [Boot packages] "tls-debug", "tmp-postgres", "tmp-proc-postgres", + "transformers", // see NOTE: [Boot packages] "ua-parser", "uniq-deep", + "unix", // see NOTE: [Boot packages] "users-postgresql-simple", "validate-input", "vector-fftw", + "visualize-type-inference", "wai-session-postgresql", + "web3-tools", "wild-bind-x11", - "Win32", + "Win32", // see NOTE: [Boot packages] "Win32-notify", "windns", "X11", "X11-xft", "x11-xim", + "xhtml", // see NOTE: [Boot packages] "xmonad", "xmonad-contrib", "xmonad-extras", diff --git a/flake.lock b/flake.lock index b86dd83..0315829 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "all-cabal-hashes": { "flake": false, "locked": { - "lastModified": 1730496291, - "narHash": "sha256-tpbpy80rGEnoewqJD6PnrBSDP7U6kiqGToDGfkn7boA=", + "lastModified": 1764068599, + "narHash": "sha256-cqiypgQN/PtGGA6JqFZ5LM/TDqeWyTRICetqGeYJTwU=", "owner": "commercialhaskell", "repo": "all-cabal-hashes", - "rev": "fa41a5f78b916fbacf406e2a49d24e8d7ff644a2", + "rev": "74615195a38035ef480f946c73735d44fa5958fe", "type": "github" }, "original": { @@ -20,11 +20,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1747046372, - "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", "owner": "edolstra", "repo": "flake-compat", - "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", "type": "github" }, "original": { @@ -38,11 +38,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1726560853, - "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -58,11 +58,11 @@ }, "locked": { "host": "gitlab.haskell.org", - "lastModified": 1730719693, - "narHash": "sha256-sA9m9T5BaDgD/UGbc+PPjbuulchu1ymcVBZEWwaC/MM=", + "lastModified": 1771125696, + "narHash": "sha256-+90XoM7PUfgD7383ki/aHC0cLKS32XVnYgI672pegcY=", "owner": "ghc", "repo": "ghc-wasm-meta", - "rev": "dee66ef6e91518a4c6af24cb2c8d96d09674467d", + "rev": "4e1f900e9933966634bc2e29dbeb81d09ce36727", "type": "gitlab" }, "original": { @@ -83,11 +83,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1745153655, - "narHash": "sha256-WilHZ5tu3OPirDN9M4ilYirgCnXnEBTf3jGyuZ+TCSE=", + "lastModified": 1773937910, + "narHash": "sha256-PisMBpM8tdlBEjIC0IYnadABcObRWZZNxkfQOT4lYmI=", "ref": "refs/heads/main", - "rev": "854fc8e4ba38893b8fb03500eb17f29dcfed79d1", - "revCount": 324, + "rev": "3f63b1bbbaea4e41d6bddb6a152935e597fc9db3", + "revCount": 359, "type": "git", "url": "https://gitlab.haskell.org/ghc/ghc.nix.git" }, @@ -120,32 +120,32 @@ }, "nixpkgs": { "locked": { - "lastModified": 1730531603, - "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", + "lastModified": 1770770419, + "narHash": "sha256-iKZMkr6Cm9JzWlRYW/VPoL0A9jVKtZYiU4zSrVeetIs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", + "rev": "6c5e707c6b5339359a9a9e215c5e66d6d802fd7a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } }, "nixpkgs_2": { "locked": { - "lastModified": 1730200266, - "narHash": "sha256-l253w0XMT8nWHGXuXqyiIC/bMvh1VRszGXgdpQlfhvU=", + "lastModified": 1769900590, + "narHash": "sha256-I7Lmgj3owOTBGuauy9FL6qdpeK2umDoe07lM4V+PnyA=", "owner": "nixos", "repo": "nixpkgs", - "rev": "807e9154dcb16384b1b765ebe9cd2bba2ac287fd", + "rev": "41e216c0ca66c83b12ab7a98cc326b5db01db646", "type": "github" }, "original": { "owner": "nixos", - "ref": "nixos-unstable", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } @@ -160,18 +160,14 @@ "nixpkgs": [ "ghc_nix", "nixpkgs" - ], - "nixpkgs-stable": [ - "ghc_nix", - "nixpkgs" ] }, "locked": { - "lastModified": 1730302582, - "narHash": "sha256-W1MIJpADXQCgosJZT8qBYLRuZls2KSiKdpnTVdKBuvU=", + "lastModified": 1763988335, + "narHash": "sha256-QlcnByMc8KBjpU37rbq5iP7Cp97HvjRP0ucfdh+M4Qc=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "af8a16fe5c264f5e9e18bcee2859b40a656876cf", + "rev": "50b9238891e388c9fdc6a5c49e49c42533a1b5ce", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index bf91a59..38be8c7 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,7 @@ pkgs = import nixpkgs { inherit system; }; - compiler = pkgs.haskell.packages.ghc9101; + compiler = pkgs.haskell.packages.ghc9122; # There are some packages that do not build well with nix: # @@ -73,6 +73,7 @@ libsepol # simple-pango libsodium # libsodium-bindings libthai # simple-pango + libwebp # webp libxml2 # c14n nettle # nettle nlopt # srtree @@ -81,10 +82,13 @@ pcre2 # simple-cairo pkg-config postgresql_16 # postgresql-libpq + libsysprof-capture # glib-stopgap + sqlite # simplest-sqlite systemdMinimal # hidapi requires udev util-linux # simple-pango requires mount xorg.libXdmcp # simple-cairo xz # lzma + z3 # LiquidHaskell ] ++ ldDeps; in { @@ -113,6 +117,12 @@ ''; }; + # Used by ci to run clc-stackage with everything and --dry-run. + ci = pkgs.mkShell { + buildInputs = deps ++ [ compiler.ghc ]; + LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath ldDeps}:$LD_LIBRARY_PATH"; + }; + dev = pkgs.mkShell { buildInputs = [ compiler.ghc diff --git a/generated/cabal.project b/generated/cabal.project index 7561378..4a2190d 100644 --- a/generated/cabal.project +++ b/generated/cabal.project @@ -1,5 +1,3 @@ -index-state: 2025-05-22T04:18:43Z - -- should be an absolute path to your custom compiler --with-compiler: /home/ghc/_build/stage1/bin/ghc @@ -7,9 +5,16 @@ packages: . optimization: False -allow-newer: - hoogle:crypton-connection, +-- The generated cabal.project.local has exact constraints for every (non-boot) +-- package in the snapshot to ensure we build the same packages every time. +-- We still want this to override individual package constraints e.g. +-- web-rep has had an overly tight bound that excludes the transformers from +-- the snapshot. The allow-newer ignores the error. +allow-newer: *:* +allow-older: *:* +constraints: bson -_old-network constraints: hlint +ghc-lib constraints: ghc-lib-parser-ex -auto +constraints: path +os-string constraints: stylish-haskell +ghc-lib diff --git a/src/builder/CLC/Stackage/Builder.hs b/src/CLC/Stackage/Builder.hs similarity index 92% rename from src/builder/CLC/Stackage/Builder.hs rename to src/CLC/Stackage/Builder.hs index 6e79c55..9dc6524 100644 --- a/src/builder/CLC/Stackage/Builder.hs +++ b/src/CLC/Stackage/Builder.hs @@ -5,6 +5,7 @@ module CLC.Stackage.Builder -- * Misc Batch.batchPackages, + Process.cabalUpdate, ) where diff --git a/src/builder/CLC/Stackage/Builder/Batch.hs b/src/CLC/Stackage/Builder/Batch.hs similarity index 100% rename from src/builder/CLC/Stackage/Builder/Batch.hs rename to src/CLC/Stackage/Builder/Batch.hs diff --git a/src/builder/CLC/Stackage/Builder/Env.hs b/src/CLC/Stackage/Builder/Env.hs similarity index 100% rename from src/builder/CLC/Stackage/Builder/Env.hs rename to src/CLC/Stackage/Builder/Env.hs diff --git a/src/builder/CLC/Stackage/Builder/Process.hs b/src/CLC/Stackage/Builder/Process.hs similarity index 88% rename from src/builder/CLC/Stackage/Builder/Process.hs rename to src/CLC/Stackage/Builder/Process.hs index 55342f5..425a4b5 100644 --- a/src/builder/CLC/Stackage/Builder/Process.hs +++ b/src/CLC/Stackage/Builder/Process.hs @@ -2,6 +2,7 @@ module CLC.Stackage.Builder.Process ( buildProject, + cabalUpdate, ) where @@ -122,6 +123,26 @@ buildProject env idx pkgs = do addPackages = Set.union pkgsSet +-- | Runs "cabal update". +cabalUpdate :: BuildEnv -> IO () +cabalUpdate env = do + Logging.putTimeInfoStr env.hLogger "Running 'cabal update'" + P.readProcessWithExitCode env.cabalPath ["update"] "" >>= \(ec, out, err) -> + case ec of + ExitSuccess -> pure () + ExitFailure _ -> do + let msg = + mconcat + [ "Failed running 'cabal update': ", + "Out: '", + T.pack out, + "', Err: '", + T.pack err, + "'" + ] + Logging.putTimeErrStr env.hLogger msg + throwIO ec + createCurrentLogsDir :: IO (OsPath, OsPath, OsPath) createCurrentLogsDir = do let dirPath = Paths.logsDir [osp|current-build|] diff --git a/src/builder/CLC/Stackage/Builder/Writer.hs b/src/CLC/Stackage/Builder/Writer.hs similarity index 100% rename from src/builder/CLC/Stackage/Builder/Writer.hs rename to src/CLC/Stackage/Builder/Writer.hs diff --git a/src/CLC/Stackage/Parser.hs b/src/CLC/Stackage/Parser.hs new file mode 100644 index 0000000..bc0669e --- /dev/null +++ b/src/CLC/Stackage/Parser.hs @@ -0,0 +1,232 @@ +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE QuasiQuotes #-} + +module CLC.Stackage.Parser + ( -- * Retrieving packages + getPackageList, + + -- * Misc helpers + printPackageList, + getPackageListByOsFmt, + ) +where + +import CLC.Stackage.Parser.API + ( StackageResponse (packages, snapshot), + ) +import CLC.Stackage.Parser.API qualified as API +import CLC.Stackage.Parser.API.CabalConfig qualified as CabalConfig +import CLC.Stackage.Parser.Utils qualified as Utils +import CLC.Stackage.Utils.Exception qualified as Ex +import CLC.Stackage.Utils.IO qualified as IO +import CLC.Stackage.Utils.JSON qualified as JSON +import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows)) +import CLC.Stackage.Utils.OS qualified as OS +import CLC.Stackage.Utils.Package (Package) +import CLC.Stackage.Utils.Package qualified as Package +import Control.Monad (when) +import Data.Aeson (FromJSON, ToJSON) +import Data.Foldable (for_) +import Data.Maybe (fromMaybe) +import Data.Set (Set) +import Data.Set qualified as Set +import Data.Text (Text) +import Data.Text qualified as T +import Data.Version (Version (Version)) +import Data.Version qualified as Vers +import GHC.Generics (Generic) +import System.Exit (ExitCode (ExitFailure, ExitSuccess)) +import System.OsPath (OsPath, osp) +import System.Process qualified as P +import Text.ParserCombinators.ReadP qualified as ReadP + +-- | Retrieves the list of packages, based on +-- 'CLC.Stackage.Parser.API.stackageUrl'. +getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] +getPackageList hLogger msnapshotPath = do + response <- getStackageResponse hLogger msnapshotPath + getPackageListByOs hLogger response OS.currentOs + +-- | Prints the package list to a file. +printPackageList :: Logging.Handle -> Maybe OsPath -> Maybe Os -> IO () +printPackageList hLogger msnapshotPath mOs = do + response <- getStackageResponse hLogger msnapshotPath + case mOs of + Just os -> printOsList response os + Nothing -> for_ [minBound .. maxBound] (printOsList response) + where + file Linux = [osp|pkgs_linux.txt|] + file Osx = [osp|pkgs_osx.txt|] + file Windows = [osp|pkgs_windows.txt|] + + printOsList response os = do + pkgs <- getPackageListByOsFmt hLogger response os + let txt = T.unlines pkgs + IO.writeFileUtf8 (file os) txt + +-- | Retrieves the package list formatted to text. +getPackageListByOsFmt :: Logging.Handle -> StackageResponse -> Os -> IO [Text] +getPackageListByOsFmt hLogger response os = do + ps <- getPackageListByOs hLogger response os + pure $ Package.toDisplayName <$> ps + +-- | Helper in case we want to see what the package set for a given OS is. +getPackageListByOs :: Logging.Handle -> StackageResponse -> Os -> IO [Package] +getPackageListByOs hLogger response os = do + excludedPkgs <- getExcludedPkgs os + let filterExcluded = flip Set.notMember excludedPkgs . (.name) + packages = filter filterExcluded response.packages + msg = + mconcat + [ "Filtered to ", + T.pack $ show $ length packages, + " packages (", + T.toLower $ T.pack $ show os, + ")." + ] + Logging.putTimeInfoStr hLogger msg + + pure packages + +getStackageResponse :: Logging.Handle -> Maybe OsPath -> IO StackageResponse +getStackageResponse hLogger msnapshotPath = do + response <- case msnapshotPath of + Nothing -> API.getStackage hLogger + Just snapshotPath -> + CabalConfig.parseCabalConfig + <$> IO.readFileUtf8 snapshotPath + + let snapshotName = fromMaybe "" response.snapshot + numPackages = length $ response.packages + numPackagesTxt = T.pack $ show numPackages + snapshotMsg = + mconcat + [ "Snapshot: ", + snapshotName, + " (", + numPackagesTxt, + " packages)" + ] + Logging.putTimeInfoStr hLogger snapshotMsg + + when (numPackages < 2000) $ do + let msg = + mconcat + [ "Only found ", + numPackagesTxt, + " packages. Is that right?" + ] + Logging.putTimeWarnStr hLogger msg + + checkGhcVersion hLogger response + + pure response + +getExcludedPkgs :: Os -> IO (Set Text) +getExcludedPkgs os = do + contents <- Ex.throwLeft . Utils.stripComments =<< IO.readBinaryFile path + + excluded <- case JSON.decode contents of + Left err -> fail err + Right x -> pure x + + pure $ Set.fromList (excluded.all ++ osSel excluded) + where + path = [osp|excluded_pkgs.jsonc|] + + osSel :: Excluded -> [Text] + osSel = case os of + Linux -> (.linux) + Osx -> (.osx) + Windows -> (.windows) + +data Excluded = MkExcluded + { all :: [Text], + linux :: [Text], + osx :: [Text], + windows :: [Text] + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) + +checkGhcVersion :: Logging.Handle -> StackageResponse -> IO () +checkGhcVersion hLogger response = do + mPathGhc <- do + -- Users are instructed to add their ghc to the PATH, so this should be + -- correct. + (ex, out, err) <- P.readProcessWithExitCode "ghc" ["--numeric-version"] "" + case ex of + ExitFailure _ -> do + let msg = "Failed running 'ghc --numeric-version': " <> T.pack err + Logging.putTimeWarnStr hLogger msg + pure Nothing + ExitSuccess -> do + let out' = T.strip $ T.pack out + case parseVers out' of + Nothing -> do + Logging.putTimeWarnStr hLogger $ "PATH ghc: Failed parsing: " <> out' + pure Nothing + Just v -> do + Logging.putTimeInfoStr hLogger $ "PATH ghc: " <> showCanonical v + pure $ Just v + + mStackageGhc <- + case response.ghc of + Nothing -> do + let msg = "Failed detecting Stackage ghc version" + Logging.putTimeWarnStr hLogger msg + pure Nothing + Just ghc -> do + case parseVers ghc of + Nothing -> do + Logging.putTimeWarnStr hLogger $ "Stackage ghc: Failed parsing: " <> ghc + pure Nothing + Just v -> do + Logging.putTimeInfoStr hLogger $ "Stackage ghc: " <> showCanonical v + pure $ Just v + + case (mPathGhc, mStackageGhc) of + (Just pathGhc, Just stackageGhc) -> case compareCanonical pathGhc stackageGhc of + GhcEq -> pure () + GhcDiffMinor -> do + let msg = "PATH and Stackage ghc have a minor difference. This may cause solver / build issues." + Logging.putTimeWarnStr hLogger msg + GhcDiffMajor -> do + let msg = "PATH and Stackage ghc have a major difference. This will likely cause solver / build issues." + Logging.putTimeWarnStr hLogger msg + _ -> pure () + where + parseVers txt = + let ps = ReadP.readP_to_S Vers.parseVersion (T.unpack txt) + in lastMaybe ps >>= versToCanonical . fst + + lastMaybe [] = Nothing + lastMaybe [x] = Just x + lastMaybe (_ : xs) = lastMaybe xs + +-- Only take the first 3 e.g. 9.12.3. This is so we can compare custom +-- ghcs (e.g. 9.12.3.20260311) against expected major.major.minor format. +versToCanonical :: Version -> Maybe (Int, Int, Int) +versToCanonical (Version bs _tags) = case bs of + (mj1 : mj2 : mn : _) -> Just (mj1, mj2, mn) + _ -> Nothing + +showCanonical :: (Int, Int, Int) -> Text +showCanonical (mj1, mj2, mn) = + T.intercalate "." $ + T.pack . show + <$> [mj1, mj2, mn] + +compareCanonical :: (Int, Int, Int) -> (Int, Int, Int) -> GhcCompareResult +compareCanonical x@(xmj1, xmj2, _xmn) y@(ymj1, ymj2, _ymn) + | x == y = GhcEq + | xmj1 /= ymj1 = GhcDiffMajor + | xmj2 /= ymj2 = GhcDiffMajor + | otherwise = GhcDiffMinor + +data GhcCompareResult + = GhcEq + | GhcDiffMinor + | GhcDiffMajor + deriving stock (Eq, Show) diff --git a/src/parser/CLC/Stackage/Parser/API.hs b/src/CLC/Stackage/Parser/API.hs similarity index 76% rename from src/parser/CLC/Stackage/Parser/API.hs rename to src/CLC/Stackage/Parser/API.hs index 6b19704..6f9fe1f 100644 --- a/src/parser/CLC/Stackage/Parser/API.hs +++ b/src/CLC/Stackage/Parser/API.hs @@ -22,7 +22,7 @@ import CLC.Stackage.Parser.API.Common ReasonStatus ), StackageException (MkStackageException), - StackageResponse (MkStackageResponse, packages), + StackageResponse (MkStackageResponse, ghc, packages, snapshot), ) import CLC.Stackage.Parser.API.JSON qualified as JSON import CLC.Stackage.Utils.Exception qualified as Ex @@ -36,7 +36,7 @@ getStackage :: Logging.Handle -> IO StackageResponse getStackage hLogger = do manager <- TLS.newTlsManager Ex.tryAny (JSON.getStackage manager stackageSnapshot) >>= \case - Right r1 -> pure $ r1 + Right r1 -> pure r1 Left jsonEx -> do let msg = mconcat @@ -48,12 +48,7 @@ getStackage hLogger = do CabalConfig.getStackage manager stackageSnapshot --- | Stackage snapshot. Note that picking a "good" snapshot is something of --- an art i.e. not all valid snapshots return json output at the --- expected endpoint. I essentially try snapshots with --- --- curl -H "Accept: application/json" -L https://stackage.org/nightly-yyyy-mm-dd --- --- until one returns json. +-- | Stackage snapshot. Currently just 'nightly' to hopefully allow clc-stackage +-- to be more flexible. stackageSnapshot :: String -stackageSnapshot = "nightly-2025-05-23" +stackageSnapshot = "nightly" diff --git a/src/parser/CLC/Stackage/Parser/API/CabalConfig.hs b/src/CLC/Stackage/Parser/API/CabalConfig.hs similarity index 52% rename from src/parser/CLC/Stackage/Parser/API/CabalConfig.hs rename to src/CLC/Stackage/Parser/API/CabalConfig.hs index de7a323..a7c285d 100644 --- a/src/parser/CLC/Stackage/Parser/API/CabalConfig.hs +++ b/src/CLC/Stackage/Parser/API/CabalConfig.hs @@ -14,14 +14,18 @@ import CLC.Stackage.Parser.API.Common ReasonStatus ), StackageException (MkStackageException), - StackageResponse (MkStackageResponse), + StackageResponse (MkStackageResponse, ghc, packages, snapshot), getStatusCode, ) import CLC.Stackage.Utils.Exception qualified as Ex import CLC.Stackage.Utils.Package qualified as Package +import Control.Applicative (asum, (<|>)) import Control.Exception (throwIO) import Control.Monad (when) -import Data.Maybe (catMaybes) +import Data.Foldable qualified as F +import Data.Maybe (fromMaybe) +import Data.Sequence (Seq) +import Data.Sequence qualified as Seq import Data.Text (Text) import Data.Text qualified as T import Data.Text.Encoding qualified as TEnc @@ -69,19 +73,81 @@ getStackage manager stackageSnapshot = do <> "/cabal.config" parseCabalConfig :: Text -> StackageResponse -parseCabalConfig = +parseCabalConfig txt = MkStackageResponse - . catMaybes - . fmap parseCabalConfigLine - . T.lines + { ghc = acc.ghc, + packages = F.toList acc.packages, + snapshot = acc.snapshot + } + where + acc = foldMap parseLine (T.lines txt) + +data Acc = MkAcc + { ghc :: Maybe Text, + packages :: Seq Package.Package, + snapshot :: Maybe Text + } + +instance Semigroup Acc where + MkAcc x1 x2 x3 <> MkAcc y1 y2 y3 = + MkAcc (x1 <|> y1) (x2 <> y2) (x3 <|> y3) + +instance Monoid Acc where + mempty = MkAcc Nothing Seq.empty Nothing + +parseLine :: Text -> Acc +parseLine txt = + fromMaybe mempty + . asum + $ ($ txt) + <$> [ parsePackage, + parseGhc, + parseSnapshot + ] + +-- | Parses a line like 'with-compiler: ghc-9.12.3. +parseGhc :: Text -> Maybe Acc +parseGhc txt = do + ghc <- stripInfixSnd "compiler: ghc-" txt + pure $ + MkAcc + { ghc = Just ghc, + packages = Seq.empty, + snapshot = Nothing + } -- | Parses a line like ' =='. -parseCabalConfigLine :: Text -> Maybe Package.Package -parseCabalConfigLine txt = do +parsePackage :: Text -> Maybe Acc +parsePackage txt = do -- Strip leading 'constraints:' keyword, if it exists. - let s = case T.stripPrefix "constraints:" txt' of - Nothing -> txt' - Just rest -> T.stripStart rest - Package.fromCabalConstraintsText s + let s = maybe txt' T.stripStart (T.stripPrefix "constraints:" txt') + p <- Package.fromCabalConstraintsText s + pure $ + MkAcc + { ghc = Nothing, + packages = Seq.singleton p, + snapshot = Nothing + } where txt' = T.stripStart txt + +-- | Parses a line like: +-- +-- 'Stackage snapshot from: http://www.stackage.org/snapshot/nightly-2026-03-25' +parseSnapshot :: Text -> Maybe Acc +parseSnapshot txt = do + snapshot <- stripInfixSnd "http://www.stackage.org/snapshot/" txt + pure $ + MkAcc + { ghc = Nothing, + packages = Seq.empty, + snapshot = Just snapshot + } + +stripInfixSnd :: Text -> Text -> Maybe Text +stripInfixSnd t1 = fmap snd . stripInfix t1 + +stripInfix :: Text -> Text -> Maybe (Text, Text) +stripInfix t1 t2 = (pre,) <$> T.stripPrefix t1 rest + where + (pre, rest) = T.breakOn t1 t2 diff --git a/src/parser/CLC/Stackage/Parser/API/Common.hs b/src/CLC/Stackage/Parser/API/Common.hs similarity index 95% rename from src/parser/CLC/Stackage/Parser/API/Common.hs rename to src/CLC/Stackage/Parser/API/Common.hs index fe2b5c9..3a225f2 100644 --- a/src/parser/CLC/Stackage/Parser/API/Common.hs +++ b/src/CLC/Stackage/Parser/API/Common.hs @@ -19,6 +19,7 @@ import Control.Exception SomeException, ) import Data.ByteString (ByteString) +import Data.Text (Text) import Data.Text.Encoding.Error (UnicodeException) import GHC.Generics (Generic) import Network.HTTP.Client (Response) @@ -27,8 +28,10 @@ import Network.HTTP.Types.Status (Status) import Network.HTTP.Types.Status qualified as Status -- | Stackage response. This type unifies different stackage responses. -newtype StackageResponse = MkStackageResponse - { packages :: [Package] +data StackageResponse = MkStackageResponse + { ghc :: Maybe Text, + packages :: [Package], + snapshot :: Maybe Text } deriving stock (Eq, Generic, Show) deriving anyclass (NFData) diff --git a/src/parser/CLC/Stackage/Parser/API/JSON.hs b/src/CLC/Stackage/Parser/API/JSON.hs similarity index 95% rename from src/parser/CLC/Stackage/Parser/API/JSON.hs rename to src/CLC/Stackage/Parser/API/JSON.hs index 9ee4a2d..9854200 100644 --- a/src/parser/CLC/Stackage/Parser/API/JSON.hs +++ b/src/CLC/Stackage/Parser/API/JSON.hs @@ -64,9 +64,11 @@ getStackage manager stackageSnapshot = do stackageUrl = "https://stackage.org/" <> stackageSnapshot toSnapshotCommon :: StackageResponse -> Common.StackageResponse -toSnapshotCommon (MkStackageResponse _ pkgs) = +toSnapshotCommon (MkStackageResponse snapshot pkgs) = Common.MkStackageResponse - { packages = toPackageCommon <$> pkgs + { ghc = Just snapshot.ghc, + packages = toPackageCommon <$> pkgs, + snapshot = Just snapshot.name } toPackageCommon :: PackageResponse -> Package.Package diff --git a/src/CLC/Stackage/Parser/Utils.hs b/src/CLC/Stackage/Parser/Utils.hs new file mode 100644 index 0000000..c19ef04 --- /dev/null +++ b/src/CLC/Stackage/Parser/Utils.hs @@ -0,0 +1,114 @@ +module CLC.Stackage.Parser.Utils + ( CommentsException (..), + stripComments, + stripInfix, + + -- * Misc + isNum, + spaceW8, + ) +where + +import Control.Exception (Exception (displayException)) +import Data.Bifunctor (second) +import Data.ByteString (ByteString) +import Data.ByteString qualified as BS +import Data.Char qualified as Ch +import Data.String (IsString) +import Data.Word (Word8) + +newtype CommentsException = MkCommentsException String + deriving stock (Show) + deriving newtype (IsString) + +instance Exception CommentsException where + displayException (MkCommentsException s) = + "Error stripping comments: " <> s + +-- | Strips a bytestring of line (//) and block (/* */) comments. +stripComments :: ByteString -> Either CommentsException ByteString +stripComments bs = + let (preComment, mWithFSlash) = BS.break (== fslashW8) bs + in case uncons2 mWithFSlash of + -- 1. Remaining text is either 1 char or a single fslash: It cannot + -- possibly start a comment, so return it. + Nothing -> Right $ preComment <> mWithFSlash + -- 2. Some remaining text. Need to check for a comment start. + -- + -- - fslashChar: a single fslash + -- - fslashNext: next char, need to check + -- - mPostCommentStart: The part after potential comment start. + Just (fslashChar, fslashNext, mPostCommentStart) + | fslashNext == fslashW8 -> + -- 2.1. Another fslash, we have started a line comment. Skip until + -- the next newline. + second + (preComment <>) + (skipLineComment mPostCommentStart >>= stripComments) + | fslashNext == starW8 -> + -- 2.2. A star, we have started a block comment. Skip until the + -- next '*/'. + second + (preComment <>) + (skipBlockComment mPostCommentStart >>= stripComments) + -- 2.3. No fslash or star i.e. not a comment start. Concat it back in, + -- and proceed with the rest of the string. + | otherwise -> + let start = preComment <> BS.pack [fslashChar, fslashNext] + in second (start <>) (stripComments mPostCommentStart) + where + -- es is the start of a line comment, w/o the opening '//'. + skipLineComment es = + let (_lineComment, mCommentEnd) = BS.break (== newlineW8) es + in case BS.uncons mCommentEnd of + -- Did not find a closing newline: error! + Nothing -> Left "Found line comment (//) without ending newline." + Just (_nl, rest) -> Right rest + + -- es is the start of a block comment, w/o the opening '/*'. + skipBlockComment es = + let (_blockComment, mCommentEnd) = BS.break (== starW8) es + in case uncons2 mCommentEnd of + -- es had fewer than 2 chars, so it could not possibly end the + -- comment, error. + Nothing -> Left "Found block comment (/*) without ending (*/)." + -- If we get here we know we have found a star char and at least one + -- other char (starNext). + Just (_starChar, starNext, mPostCommentStart) + -- Found an ending slash, we have successfully ended the comment. + | starNext == fslashW8 -> Right mPostCommentStart + -- The next char is another star. Need to add it back in, in + -- case it is part of the ending */ e.g. we have /***/. + | starNext == starW8 -> skipBlockComment (BS.cons starW8 mPostCommentStart) + -- starNext is something else. Search the rest of the string. + | otherwise -> skipBlockComment mPostCommentStart + +uncons2 :: ByteString -> Maybe (Word8, Word8, ByteString) +uncons2 bs = case BS.uncons bs of + Nothing -> Nothing + Just (c1, rest1) -> case BS.uncons rest1 of + Nothing -> Nothing + Just (c2, rest2) -> Just (c1, c2, rest2) + +newlineW8 :: Word8 +newlineW8 = i2w8 $ Ch.ord '\n' + +starW8 :: Word8 +starW8 = i2w8 $ Ch.ord '*' + +fslashW8 :: Word8 +fslashW8 = i2w8 $ Ch.ord '/' + +spaceW8 :: Word8 +spaceW8 = i2w8 $ Ch.ord ' ' + +isNum :: Word8 -> Bool +isNum w = w >= (i2w8 $ Ch.ord '0') && w <= (i2w8 $ Ch.ord '9') + +i2w8 :: Int -> Word8 +i2w8 = fromIntegral + +stripInfix :: ByteString -> ByteString -> Maybe (ByteString, ByteString) +stripInfix bs1 bs2 = (pre,) <$> BS.stripPrefix bs1 rest + where + (pre, rest) = BS.breakSubstring bs1 bs2 diff --git a/src/runner/CLC/Stackage/Runner.hs b/src/CLC/Stackage/Runner.hs similarity index 65% rename from src/runner/CLC/Stackage/Runner.hs rename to src/CLC/Stackage/Runner.hs index c2bcfb2..1ade3d5 100644 --- a/src/runner/CLC/Stackage/Runner.hs +++ b/src/CLC/Stackage/Runner.hs @@ -8,7 +8,7 @@ where import CLC.Stackage.Builder qualified as Builder import CLC.Stackage.Builder.Env - ( BuildEnv (progress), + ( BuildEnv (hLogger, progress), Progress (failuresRef), ) import CLC.Stackage.Builder.Writer qualified as Writer @@ -17,10 +17,11 @@ import CLC.Stackage.Runner.Env qualified as Env import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.Package (Package) import Control.Exception (bracket, throwIO) -import Control.Monad (when) +import Control.Monad (unless, when) import Data.Foldable (for_) import Data.IORef (readIORef) import System.Exit (ExitCode (ExitFailure)) +import System.IO qualified as IO -- | Entry-point for testing clc-stackage. In particular: -- @@ -39,7 +40,7 @@ run hLogger = runModifyPackages hLogger id -- | Like 'run', except takes a package modifier. This is used for testing, so -- that we can whittle down the (very large) package set. runModifyPackages :: Logging.Handle -> ([Package] -> [Package]) -> IO () -runModifyPackages hLogger modifyPackages = do +runModifyPackages hLogger modifyPackages = withHiddenInput $ do bracket (Env.setup hLogger modifyPackages) Env.teardown $ \env -> do let buildEnv = env.buildEnv pkgGroupsIdx = Builder.batchPackages buildEnv @@ -47,7 +48,30 @@ runModifyPackages hLogger modifyPackages = do -- write the entire package set to the cabal.project.local's constraints Writer.writeCabalProjectLocal env.completePackageSet + unless env.noCabalUpdate $ Builder.cabalUpdate buildEnv + + Logging.putTimeInfoStr buildEnv.hLogger "Starting build(s)" + for_ pkgGroupsIdx $ \(pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup numErrors <- length <$> readIORef buildEnv.progress.failuresRef when (numErrors > 0) $ throwIO $ ExitFailure 1 + +-- | Hides stdin, useful so that accidental key presses do not overwrite logs. +withHiddenInput :: IO a -> IO a +withHiddenInput m = bracket hideInput unhideInput (const m) + where + -- Note that this may not work on windows. + -- + -- - https://stackoverflow.com/questions/15848975/preventing-input-characters-appearing-in-terminal + -- - https://hackage.haskell.org/package/echo + hideInput = do + buffMode <- IO.hGetBuffering IO.stdin + echoMode <- IO.hGetEcho IO.stdin + IO.hSetBuffering IO.stdin IO.NoBuffering + IO.hSetEcho IO.stdin False + pure (buffMode, echoMode) + + unhideInput (buffMode, echoMode) = do + IO.hSetBuffering IO.stdin buffMode + IO.hSetEcho IO.stdin echoMode diff --git a/src/runner/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs similarity index 56% rename from src/runner/CLC/Stackage/Runner/Args.hs rename to src/CLC/Stackage/Runner/Args.hs index 9652b04..431a25d 100644 --- a/src/runner/CLC/Stackage/Runner/Args.hs +++ b/src/CLC/Stackage/Runner/Args.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE OverloadedLists #-} + module CLC.Stackage.Runner.Args ( Args (..), ColorLogs (..), @@ -8,9 +10,16 @@ where import CLC.Stackage.Builder.Env ( WriteLogs (WriteLogsCurrent, WriteLogsNone, WriteLogsSaveFailures), ) +import CLC.Stackage.Utils.Paths qualified as Paths +import Control.Exception (IOException, try) +import Data.Either (fromRight) +import Data.List qualified as L +import Data.List.NonEmpty (NonEmpty ((:|))) import Data.String qualified as Str +import GHC.IO.Exception (ExitCode (ExitFailure, ExitSuccess)) import Options.Applicative ( Mod, + OptionFields, Parser, ParserInfo ( ParserInfo, @@ -25,13 +34,17 @@ import Options.Applicative (<**>), ) import Options.Applicative qualified as OA +import Options.Applicative.Builder.Completer (Completer) +import Options.Applicative.Builder.Completer qualified as OAC import Options.Applicative.Help (Doc) import Options.Applicative.Help.Chunk (Chunk (Chunk)) import Options.Applicative.Help.Chunk qualified as Chunk import Options.Applicative.Help.Pretty qualified as Pretty import Options.Applicative.Types (ArgPolicy (Intersperse), ReadM) +import System.Directory.OsPath qualified as Dir import System.OsPath (OsPath) import System.OsPath qualified as OsP +import System.Process qualified as P -- | Log coloring option. data ColorLogs @@ -57,6 +70,8 @@ data Args = MkArgs -- | If true, the first group that fails to completely build stops -- clc-stackage. groupFailFast :: Bool, + -- | If true, the 'cabal update' step is skipped. + noCabalUpdate :: Bool, -- | Disables the cache, which otherwise saves the outcome of a run in a -- json file. The cache is used for resuming a run that was interrupted. noCache :: Bool, @@ -65,6 +80,8 @@ data Args = MkArgs -- | If true, the first package that fails _within_ a package group will -- cause the entire group to fail. packageFailFast :: Bool, + -- | If true, prints the package set that will be used, and exits. + printPackageSet :: Bool, -- | Whether to retry packages that failed. retryFailures :: Bool, -- | Optional path to snapshot file. If given, we use the file's contents @@ -90,7 +107,9 @@ getArgs = OA.execParser parserInfoArgs infoPolicy = Intersperse } headerTxt = Just "clc-stackage: Builds all packages in a stackage snapshot." - desc = + desc = intro <> completions <> examples + + intro = Chunk.vsepChunks [ Chunk.paragraph $ mconcat @@ -108,50 +127,72 @@ getArgs = OA.execParser parserInfoArgs "off." ], Chunk.paragraph "Alternatively, to build everything in one go, run:", - Pretty.indent 2 <$> Chunk.paragraph "clc-stackage", + Pretty.indent 2 <$> Chunk.paragraph "$ clc-stackage", Chunk.paragraph $ mconcat [ "This will build everything in one package group, and pass ", "--keep-going to cabal." - ], + ] + ] + + completions = + Chunk.vsepChunks + [ line, + Chunk.paragraph "Shell completions are available e.g.", + Pretty.indent 2 <$> Chunk.stringChunk "$ source <(clc-stackage --bash-completion-script `which clc-stackage`)" + ] + + examples = + Chunk.vsepChunks + [ line, Chunk.paragraph "Examples:", mkExample - [ "# Basic example", + [ "1. Basic example:", + "", "$ clc-stackage" ], mkExample - [ "# Batch with groups of 100 and some cabal options", + [ "2. Batch with groups of 100 and some cabal options:", + "", "$ clc-stackage --batch 100 --cabal-options='--semaphore --verbose=1'" ], mkExample - [ "# Run with custom cabal", - "$ clc-stackage --cabal-path=path/to/cabal --cabal-global-options='--store-dir=path/to/store'" + [ "3. Run with custom cabal:", + "", + "$ clc-stackage \\", + " --cabal-path=path/to/cabal \\", + " --cabal-global-options='--store-dir=path/to/store'" ], mkExample - [ "# Run with custom snapshot", + [ "4. Run with custom snapshot:", + "", "$ clc-stackage --snapshot-path=path/to/snapshot-file" ] ] - mkExample :: [String] -> Chunk Doc - mkExample = + + mkExample :: NonEmpty String -> Chunk Doc + mkExample = identPara 2 5 + + identPara :: Int -> Int -> NonEmpty String -> Chunk Doc + identPara hIndent lIndent (h :| xs) = Chunk.vcatChunks - . fmap (fmap (Pretty.indent 2) . Chunk.stringChunk) + . (\ys -> toChunk hIndent h : ys) + . fmap (toChunk lIndent) + $ xs + + toChunk _ "" = line + toChunk i other = fmap (Pretty.indent i) . Chunk.stringChunk $ other + + line = Chunk (Just Pretty.softline) parseCliArgs :: Parser Args parseCliArgs = ( do - batch <- parseBatch - cabalGlobalOpts <- parseCabalGlobalOpts - cabalOpts <- parseCabalOpts - cabalPath <- parseCabalPath - colorLogs <- parseColorLogs - groupFailFast <- parseGroupFailFast - noCache <- parseNoCache - noCleanup <- parseNoCleanup - packageFailFast <- parsePackageFailFast - retryFailures <- parseRetryFailures - snapshotPath <- parseSnapshotPath - writeLogs <- parseWriteLogs + ~(cabalGlobalOpts, cabalOpts, cabalPath, noCabalUpdate) <- parseCabalGroup + ~(noCache, retryFailures) <- parseCacheGroup + ~(groupFailFast, packageFailFast) <- parseFailuresGroup + ~(batch, printPackageSet, snapshotPath) <- parseMiscGroup + ~(colorLogs, noCleanup, writeLogs) <- parseOutputGroup pure $ MkArgs @@ -161,15 +202,51 @@ parseCliArgs = cabalPath, colorLogs, groupFailFast, + noCabalUpdate, noCache, noCleanup, packageFailFast, + printPackageSet, retryFailures, snapshotPath, writeLogs } ) <**> OA.helper + where + parseCabalGroup = + OA.parserOptionGroup "Cabal options:" $ + (,,,) + <$> parseCabalGlobalOpts + <*> parseCabalOpts + <*> parseCabalPath + <*> parseNoCabalUpdate + + parseCacheGroup = + OA.parserOptionGroup "Cache options:" $ + (,) + <$> parseNoCache + <*> parseRetryFailures + + parseFailuresGroup = + OA.parserOptionGroup "Failure options:" $ + (,) + <$> parseGroupFailFast + <*> parsePackageFailFast + + parseMiscGroup = + OA.parserOptionGroup "Misc options:" $ + (,,) + <$> parseBatch + <*> parsePrintPackageSet + <*> parseSnapshotPath + + parseOutputGroup = + OA.parserOptionGroup "Output options:" $ + (,,) + <$> parseColorLogs + <*> parseNoCleanup + <*> parseWriteLogs parseBatch :: Parser (Maybe Int) parseBatch = @@ -181,10 +258,10 @@ parseBatch = OA.metavar "NAT", mkHelp $ mconcat - [ "If given N, divides the package set into groups of at ", + [ "Divides the package set into groups of at ", "most size N. This can be useful when building everything ", - "in one build is infeasible, or taking advantage of the ", - "better status reporting. No option means we batch ", + "in one build is infeasible, or we want to take advantage of ", + "the better status reporting. No option means we batch ", "everything in the same group." ] ] @@ -230,6 +307,7 @@ parseCabalPath = ( mconcat [ OA.long "cabal-path", OA.metavar "PATH", + OA.completer compgenCwdPathsCompleter, mkHelp "Optional path to cabal executable." ] ) @@ -240,7 +318,8 @@ parseColorLogs = readColorLogs ( mconcat [ OA.long "color-logs", - OA.metavar "(off | on | detect)", + OA.metavar "(detect | on | off)", + OA.completeWith ["detect", "on", "off"], OA.value ColorLogsDetect, mkHelp "Determines whether we color logs. Defaults to detect." ] @@ -251,7 +330,7 @@ parseColorLogs = "off" -> pure ColorLogsOff "on" -> pure ColorLogsOn "detect" -> pure ColorLogsDetect - bad -> fail $ "Expected one of (off | on | detect), received: " <> bad + bad -> fail $ "Expected one of (detect | on | off), received: " <> bad parseGroupFailFast :: Parser Bool parseGroupFailFast = @@ -264,10 +343,21 @@ parseGroupFailFast = where helpTxt = mconcat - [ "If true, the first group that fails to completely build stops ", + [ "If true, the first batch group that fails to completely build stops ", "clc-stackage." ] +parseNoCabalUpdate :: Parser Bool +parseNoCabalUpdate = + OA.switch + ( mconcat + [ OA.long "no-cabal-update", + mkHelpNoLine helpTxt + ] + ) + where + helpTxt = "If true, skips the 'cabal update' step." + parseNoCache :: Parser Bool parseNoCache = OA.switch @@ -297,24 +387,39 @@ parsePackageFailFast = OA.switch ( mconcat [ OA.long "package-fail-fast", - mkHelp helpTxt + mkHelpNoLine helpTxt ] ) where helpTxt = mconcat - [ "If true, the first package that fails _within_ a package group ", + [ "If true, the first package that fails _within_ a batch group ", "will cause the entire group to fail. We then move to the next ", "group, as normal. The default (off) behavior is equivalent to ", "cabal's --keep-going)." ] +parsePrintPackageSet :: Parser Bool +parsePrintPackageSet = + OA.switch + ( mconcat + [ OA.long "print-package-set", + mkHelp helpTxt + ] + ) + where + helpTxt = + mconcat + [ "Instead of running the builder, prints the package set that will ", + "be used and exits." + ] + parseRetryFailures :: Parser Bool parseRetryFailures = OA.switch ( mconcat [ OA.long "retry-failures", - mkHelp "Retries failures from the cache. Incompatible with --no-cache. " + mkHelpNoLine "Retries failures from the cache. Incompatible with --no-cache. " ] ) @@ -326,16 +431,17 @@ parseSnapshotPath = ( mconcat [ OA.long "snapshot-path", OA.metavar "PATH", - mkHelp $ + OA.completer compgenCwdPathsCompleter, + mkHelpNoLine $ mconcat - [ "Optional path to snapshot file. If given, this overrides ", + [ "Optional path to snapshot file. This overrides ", "the stackage snapshot; that is, we use the file's contents, ", "rather than the stackage server. The file should be ", "formatted similar to ", "https://www.stackage.org//cabal.config i.e. each ", "line should be ' ==' e.g. 'lens ==5.3.4'. Note ", "that the snapshot is still filtered according to ", - "excluded_pkgs.json." + "excluded_pkgs.jsonc." ] ] ) @@ -347,31 +453,32 @@ parseWriteLogs = readWriteLogs ( mconcat [ OA.long "write-logs", - OA.metavar "(none | current | save-failures)", - mkHelp $ - mconcat - [ "Determines what cabal logs to write to the output/ ", - "directory. 'None' writes nothing. 'Current' writes stdout ", - "and stderr for the currently building project. ", - "'Save-failures' is the same as 'current' except the files ", - "are not deleted if the build failed. Defaults to ", - "save-failures." - ] + OA.metavar "(current | save-failures | off)", + OA.completeWith ["current", "save-failures", "off"], + helpTxt ] ) where readWriteLogs = OA.str >>= \case - "none" -> pure WriteLogsNone + "off" -> pure WriteLogsNone "current" -> pure WriteLogsCurrent "save-failures" -> pure WriteLogsSaveFailures other -> fail $ mconcat - [ "Expected one of (none | current | save-failures), received: ", + [ "Expected one of (current | save-failures | off), received: ", other ] + helpTxt = + itemizeNoLine + [ "Determines what cabal logs to write to the output/ directory. 'save-failures' is the default.", + "current: Writes stdout and sdterr for the currently building project.", + "save-failures: Same as 'current', except the files are not deleted if the build failed.", + "off: Writes nothing." + ] + readOsPath :: ReadM OsPath readOsPath = do fp <- OA.str @@ -385,3 +492,72 @@ mkHelp = . fmap (<> Pretty.hardline) . Chunk.unChunk . Chunk.paragraph + +-- The last entry in each option group should use this, to prevent an extra +-- new line. +mkHelpNoLine :: String -> Mod f a +mkHelpNoLine = + OA.helpDoc + . Chunk.unChunk + . Chunk.paragraph + +-- | 'itemize' that does not append a trailing newline. Useful for the last +-- option in a group, as groups already start a newline. +itemizeNoLine :: NonEmpty String -> Mod OptionFields a +itemizeNoLine = + OA.helpDoc + . Chunk.unChunk + . itemizeHelper + +itemizeHelper :: NonEmpty String -> Chunk Doc +itemizeHelper (intro :| ds) = + Chunk.vcatChunks $ + Chunk.paragraph intro + : toChunk Pretty.softline + : (toItem <$> ds) + where + toItem d = + fmap (Pretty.nest 2) + . Chunk.paragraph + $ ("- " <> d) + + toChunk :: a -> Chunk a + toChunk = Chunk . Just + +-- | Paths completer that tries compgen first, then falls back to +-- directory. +compgenCwdPathsCompleter :: Completer +compgenCwdPathsCompleter = bashCompleterQuiet "file" <> cwdPathsCompleter + +-- | Like optparse's bashCompleter, except this does not report completions +-- errors, which can otherwise make the output difficult to read. +bashCompleterQuiet :: String -> Completer +bashCompleterQuiet action = OAC.mkCompleter $ \word -> do + let cmd = L.unwords ["compgen", "-A", action, "--", OAC.requote word] + (ec, out, _err) <- P.readCreateProcessWithExitCode (P.shell cmd) "" + pure $ case ec of + ExitFailure _ -> [] + ExitSuccess -> L.lines out + +-- | Paths completer that uses directory. +cwdPathsCompleter :: Completer +cwdPathsCompleter = OAC.mkCompleter $ \word -> do + eFiles <- tryIO $ do + cwd <- Dir.getCurrentDirectory + Dir.listDirectory cwd + + let files = fromRight [] eFiles + + pure $ foldr (go word) [] files + where + go :: String -> OsPath -> [String] -> [String] + go word p acc = do + let pStr = Paths.decodeUtfLenient p + matchesPat = word `L.isPrefixOf` pStr + + if matchesPat + then pStr : acc + else acc + +tryIO :: IO a -> IO (Either IOException a) +tryIO = try diff --git a/src/runner/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs similarity index 96% rename from src/runner/CLC/Stackage/Runner/Env.hs rename to src/CLC/Stackage/Runner/Env.hs index 2bcd26b..d5b3104 100644 --- a/src/runner/CLC/Stackage/Runner/Env.hs +++ b/src/CLC/Stackage/Runner/Env.hs @@ -46,7 +46,7 @@ import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.Package (Package (MkPackage, name, version)) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) -import Control.Monad (join, unless) +import Control.Monad (join, unless, when) import Data.Bool (Bool (False, True), not) import Data.Foldable (Foldable (foldl')) import Data.IORef (newIORef, readIORef) @@ -73,6 +73,8 @@ data RunnerEnv = MkRunnerEnv -- cabal.project.local's constraint section, to ensure we always use the -- same transitive dependencies. completePackageSet :: [Package], + -- | Disables the 'cabal update' step. + noCabalUpdate :: Bool, -- | Disables the cache, which otherwise saves the outcome of a run in a -- json file. The cache is used for resuming a run that was interrupted. noCache :: Bool, @@ -101,6 +103,11 @@ setup hLoggerRaw modifyPackages = do -- Update logger with CLI color param. let hLogger = hLoggerRaw {Logging.color = colorLogs} + when cliArgs.printPackageSet $ do + Logging.putTimeInfoStr hLogger "Printing package set" + Parser.printPackageList hLogger cliArgs.snapshotPath Nothing + throwIO ExitSuccess + -- Set up build args for cabal, filling in missing defaults let buildArgs = join @@ -192,6 +199,7 @@ setup hLoggerRaw modifyPackages = do { buildEnv, cache, completePackageSet, + noCabalUpdate = cliArgs.noCabalUpdate, noCache = cliArgs.noCache, noCleanup = cliArgs.noCleanup, retryFailures = cliArgs.retryFailures, diff --git a/src/runner/CLC/Stackage/Runner/Report.hs b/src/CLC/Stackage/Runner/Report.hs similarity index 100% rename from src/runner/CLC/Stackage/Runner/Report.hs rename to src/CLC/Stackage/Runner/Report.hs diff --git a/src/utils/CLC/Stackage/Utils/Exception.hs b/src/CLC/Stackage/Utils/Exception.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/Exception.hs rename to src/CLC/Stackage/Utils/Exception.hs diff --git a/src/utils/CLC/Stackage/Utils/IO.hs b/src/CLC/Stackage/Utils/IO.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/IO.hs rename to src/CLC/Stackage/Utils/IO.hs diff --git a/src/utils/CLC/Stackage/Utils/JSON.hs b/src/CLC/Stackage/Utils/JSON.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/JSON.hs rename to src/CLC/Stackage/Utils/JSON.hs diff --git a/src/utils/CLC/Stackage/Utils/Logging.hs b/src/CLC/Stackage/Utils/Logging.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/Logging.hs rename to src/CLC/Stackage/Utils/Logging.hs diff --git a/src/utils/CLC/Stackage/Utils/OS.hs b/src/CLC/Stackage/Utils/OS.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/OS.hs rename to src/CLC/Stackage/Utils/OS.hs diff --git a/src/utils/CLC/Stackage/Utils/Package.hs b/src/CLC/Stackage/Utils/Package.hs similarity index 99% rename from src/utils/CLC/Stackage/Utils/Package.hs rename to src/CLC/Stackage/Utils/Package.hs index 7e91e08..68e1806 100644 --- a/src/utils/CLC/Stackage/Utils/Package.hs +++ b/src/CLC/Stackage/Utils/Package.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE ViewPatterns #-} - -- | Provides the type representing a package with version. module CLC.Stackage.Utils.Package ( -- * Package diff --git a/src/utils/CLC/Stackage/Utils/Paths.hs b/src/CLC/Stackage/Utils/Paths.hs similarity index 100% rename from src/utils/CLC/Stackage/Utils/Paths.hs rename to src/CLC/Stackage/Utils/Paths.hs diff --git a/src/parser/CLC/Stackage/Parser.hs b/src/parser/CLC/Stackage/Parser.hs deleted file mode 100644 index b92bf86..0000000 --- a/src/parser/CLC/Stackage/Parser.hs +++ /dev/null @@ -1,114 +0,0 @@ -{-# LANGUAGE QuasiQuotes #-} - -module CLC.Stackage.Parser - ( -- * Retrieving packages - getPackageList, - - -- * Misc helpers - printPackageList, - getPackageListByOsFmt, - ) -where - -import CLC.Stackage.Parser.API - ( StackageResponse (packages), - ) -import CLC.Stackage.Parser.API qualified as API -import CLC.Stackage.Parser.API.CabalConfig qualified as CabalConfig -import CLC.Stackage.Utils.IO qualified as IO -import CLC.Stackage.Utils.JSON qualified as JSON -import CLC.Stackage.Utils.Logging qualified as Logging -import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows)) -import CLC.Stackage.Utils.OS qualified as OS -import CLC.Stackage.Utils.Package (Package) -import CLC.Stackage.Utils.Package qualified as Package -import Control.Monad (when) -import Data.Aeson (FromJSON, ToJSON) -import Data.Foldable (for_) -import Data.Set (Set) -import Data.Set qualified as Set -import Data.Text (Text) -import Data.Text qualified as T -import GHC.Generics (Generic) -import System.OsPath (OsPath, osp) - --- | Retrieves the list of packages, based on --- 'CLC.Stackage.Parser.API.stackageUrl'. -getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] -getPackageList hLogger msnapshotPath = - getPackageListByOs hLogger msnapshotPath OS.currentOs - --- | Prints the package list to a file. -printPackageList :: Maybe Os -> IO () -printPackageList mOs = do - case mOs of - Just os -> printOsList os - Nothing -> for_ [minBound .. maxBound] printOsList - where - file Linux = [osp|pkgs_linux.txt|] - file Osx = [osp|pkgs_osx.txt|] - file Windows = [osp|pkgs_windows.txt|] - - printOsList os = do - pkgs <- getPackageListByOsFmt os - let txt = T.unlines pkgs - IO.writeFileUtf8 (file os) txt - --- | Retrieves the package list formatted to text. -getPackageListByOsFmt :: Os -> IO [Text] -getPackageListByOsFmt = - (fmap . fmap) Package.toDisplayName - . getPackageListByOs Logging.mkDefaultLogger Nothing - --- | Helper in case we want to see what the package set for a given OS is. -getPackageListByOs :: Logging.Handle -> Maybe OsPath -> Os -> IO [Package] -getPackageListByOs hLogger msnapshotPath os = do - excludedPkgs <- getExcludedPkgs os - let filterExcluded = flip Set.notMember excludedPkgs . (.name) - - response <- case msnapshotPath of - Nothing -> API.getStackage hLogger - Just snapshotPath -> - CabalConfig.parseCabalConfig - <$> IO.readFileUtf8 snapshotPath - - let numPackages = length response.packages - when (numPackages < 2000) $ do - let msg = - mconcat - [ "Only found ", - T.pack $ show numPackages, - " packages. Is that right?" - ] - Logging.putTimeWarnStr hLogger msg - - let packages = filter filterExcluded response.packages - - pure packages - -getExcludedPkgs :: Os -> IO (Set Text) -getExcludedPkgs os = do - contents <- IO.readBinaryFile path - - excluded <- case JSON.decode contents of - Left err -> fail err - Right x -> pure x - - pure $ Set.fromList (excluded.all ++ osSel excluded) - where - path = [osp|excluded_pkgs.json|] - - osSel :: Excluded -> [Text] - osSel = case os of - Linux -> (.linux) - Osx -> (.osx) - Windows -> (.windows) - -data Excluded = MkExcluded - { all :: [Text], - linux :: [Text], - osx :: [Text], - windows :: [Text] - } - deriving stock (Eq, Generic, Show) - deriving anyclass (FromJSON, ToJSON) diff --git a/test/functional/Main.hs b/test/functional/Main.hs index bf90e24..7ddb8a1 100644 --- a/test/functional/Main.hs +++ b/test/functional/Main.hs @@ -2,17 +2,20 @@ module Main (main) where +import CLC.Stackage.Parser.Utils qualified as Parser.Utils import CLC.Stackage.Runner qualified as Runner import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging -import CLC.Stackage.Utils.OS (Os (Windows), currentOs) +import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows), currentOs) import CLC.Stackage.Utils.Package (Package (name)) import CLC.Stackage.Utils.Paths qualified as Paths +import Control.Applicative (asum) import Data.ByteString (ByteString) +import Data.ByteString qualified as BS import Data.ByteString.Char8 qualified as C8 import Data.IORef (IORef, modifyIORef', newIORef, readIORef) import Data.List qualified as L -import Data.Maybe (isJust) +import Data.Maybe (fromMaybe, isJust) import Data.Set qualified as Set import Data.Text (Text) import Data.Text.Encoding qualified as TEnc @@ -98,7 +101,7 @@ modifyPackages = filter (\p -> Set.member p.name pkgs) Set.fromList [ "cborg", "clock", - "mtl", + "extra", "optics-core", "profunctors" ] @@ -165,11 +168,45 @@ runGolden getNoCleanup params = finalArgs = args' ++ noCleanupArgs logs <- withArgs finalArgs params.runner + let logs' = filter (not . skipLog) $ fmap massageLogs logs - writeActualFile $ toBS logs + writeActualFile $ toBS logs' where + -- Logs whose presence is non-deterministic i.e. need to be removed + -- entirely. + skipLog = BS.isInfixOf "PATH and Stackage ghc" + + -- Strip non-determinism from logs (e.g. version numbers, snapshots). + massageLogs bs = + fromMaybe bs $ + asum $ fmap ($ bs) + [ fixGhcStr, + fixSnapshotStr, + fixNumPkgs + ] + where + fixGhcStr b = do + (pre, r1) <- Parser.Utils.stripInfix "ghc: " b + let (_vers, rest) = BS.break (== Parser.Utils.spaceW8) r1 + pure $ pre <> "ghc: " <> rest + + fixSnapshotStr b = do + (pre, r1) <- Parser.Utils.stripInfix "Snapshot: " b + let (_vers, r2) = BS.break (== Parser.Utils.spaceW8) r1 + rest = fromMaybe r2 (fixNumPkgs r2) + pure $ pre <> "Snapshot: " <> rest + + fixNumPkgs b = do + (r1, post) <- Parser.Utils.stripInfix " packages" b + let (pre, _num) = BS.breakEnd (not . Parser.Utils.isNum) r1 + pure $ pre <> " packages" <> post + -- test w/ color off since CI can't handle it, apparently - args' = "--color-logs" : "off" : params.args + args' = + "--color-logs" + : "off" + : "--no-cabal-update" + : params.args baseTestPath = goldensDir @@ -190,5 +227,6 @@ goldensDir = [osp|test|] [osp|functional|] [osp|goldens|] ext :: OsPath ext = case currentOs of + Linux -> [osp|_linux|] + Osx -> [osp|_osx|] Windows -> [osp|_windows|] - _ -> [osp|_posix|] diff --git a/test/functional/goldens/testSmallBatch_linux.golden b/test/functional/goldens/testSmallBatch_linux.golden new file mode 100644 index 0000000..0927d98 --- /dev/null +++ b/test/functional/goldens/testSmallBatch_linux.golden @@ -0,0 +1,18 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (linux). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 +[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 +[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmallBatch_osx.golden b/test/functional/goldens/testSmallBatch_osx.golden new file mode 100644 index 0000000..4010f18 --- /dev/null +++ b/test/functional/goldens/testSmallBatch_osx.golden @@ -0,0 +1,18 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (osx). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 +[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 +[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmallBatch_posix.golden b/test/functional/goldens/testSmallBatch_posix.golden deleted file mode 100644 index e8ce0c3..0000000 --- a/test/functional/goldens/testSmallBatch_posix.golden +++ /dev/null @@ -1,13 +0,0 @@ -[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: mtl-2.3.1, optics-core-0.4.1.1 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.2 - - -- Successes: 5 (100%) -- Failures: 0 (0%) -- Untested: 0 (0%) - -- Start: 2020-05-31 12:00:00 -- End: 2020-05-31 12:00:00 - diff --git a/test/functional/goldens/testSmallBatch_windows.golden b/test/functional/goldens/testSmallBatch_windows.golden index 5569fee..9156c72 100644 --- a/test/functional/goldens/testSmallBatch_windows.golden +++ b/test/functional/goldens/testSmallBatch_windows.golden @@ -1,7 +1,12 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (windows). +[2020-05-31 12:00:00][Info] Starting build(s) [2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: mtl-2.3.1, optics-core-0.4.1.1 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.2 +[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 +[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_linux.golden b/test/functional/goldens/testSmallSnapshotPath_linux.golden new file mode 100644 index 0000000..6fea1a1 --- /dev/null +++ b/test/functional/goldens/testSmallSnapshotPath_linux.golden @@ -0,0 +1,17 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Warn] Only found packages. Is that right? +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (linux). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmallSnapshotPath_osx.golden b/test/functional/goldens/testSmallSnapshotPath_osx.golden new file mode 100644 index 0000000..e360336 --- /dev/null +++ b/test/functional/goldens/testSmallSnapshotPath_osx.golden @@ -0,0 +1,17 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Warn] Only found packages. Is that right? +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (osx). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmallSnapshotPath_posix.golden b/test/functional/goldens/testSmallSnapshotPath_posix.golden deleted file mode 100644 index 731f562..0000000 --- a/test/functional/goldens/testSmallSnapshotPath_posix.golden +++ /dev/null @@ -1,12 +0,0 @@ -[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json -[2020-05-31 12:00:00][Warn] Only found 8 packages. Is that right? -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-installed... - - -- Successes: 5 (100%) -- Failures: 0 (0%) -- Untested: 0 (0%) - -- Start: 2020-05-31 12:00:00 -- End: 2020-05-31 12:00:00 - diff --git a/test/functional/goldens/testSmallSnapshotPath_windows.golden b/test/functional/goldens/testSmallSnapshotPath_windows.golden index 6ba7a5a..46d4a5c 100644 --- a/test/functional/goldens/testSmallSnapshotPath_windows.golden +++ b/test/functional/goldens/testSmallSnapshotPath_windows.golden @@ -1,6 +1,11 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json -[2020-05-31 12:00:00][Warn] Only found 8 packages. Is that right? -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-installed... +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Warn] Only found packages. Is that right? +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (windows). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_linux.golden b/test/functional/goldens/testSmall_linux.golden new file mode 100644 index 0000000..9dddebd --- /dev/null +++ b/test/functional/goldens/testSmall_linux.golden @@ -0,0 +1,16 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (linux). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmall_osx.golden b/test/functional/goldens/testSmall_osx.golden new file mode 100644 index 0000000..5d67dff --- /dev/null +++ b/test/functional/goldens/testSmall_osx.golden @@ -0,0 +1,16 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (osx). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmall_posix.golden b/test/functional/goldens/testSmall_posix.golden deleted file mode 100644 index 9e6ac70..0000000 --- a/test/functional/goldens/testSmall_posix.golden +++ /dev/null @@ -1,11 +0,0 @@ -[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-2.3.1, op... - - -- Successes: 5 (100%) -- Failures: 0 (0%) -- Untested: 0 (0%) - -- Start: 2020-05-31 12:00:00 -- End: 2020-05-31 12:00:00 - diff --git a/test/functional/goldens/testSmall_windows.golden b/test/functional/goldens/testSmall_windows.golden index 519cd25..6c649cc 100644 --- a/test/functional/goldens/testSmall_windows.golden +++ b/test/functional/goldens/testSmall_windows.golden @@ -1,5 +1,10 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-2.3.1, op... +[2020-05-31 12:00:00][Info] Snapshot: ( packages) +[2020-05-31 12:00:00][Info] PATH ghc: +[2020-05-31 12:00:00][Info] Stackage ghc: +[2020-05-31 12:00:00][Info] Filtered to packages (windows). +[2020-05-31 12:00:00][Info] Starting build(s) +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... - Successes: 5 (100%) diff --git a/test/functional/snapshot.txt b/test/functional/snapshot.txt index f1f48cb..460a4ed 100644 --- a/test/functional/snapshot.txt +++ b/test/functional/snapshot.txt @@ -1,10 +1,13 @@ -# This is the list of packages from the functional tests' modifyPackages -# (chosen for consistency) with a few others thrown in, to test that filtering -# works the same for both querying stackage and reading this file. This list -# is intentionally kept small to make maintenance easier. +-- This is the list of packages from the functional tests' modifyPackages +-- (chosen for consistency) with a few others thrown in, to test that filtering +-- works the same for both querying stackage and reading this file. This list +-- is intentionally kept small to make maintenance easier. +-- Stackage snapshot from: http://www.stackage.org/snapshot/nightly-2026-03-25 +-- with-compiler: ghc-9.12.3 aeson ==2.2.3.0 cborg ==0.2.10.0 clock ==0.8.4 +extra ==1.8 kan-extensions ==5.2.6 mtl installed optics-core ==0.4.1.1 diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index a360cb9..2d7f94d 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -25,6 +25,7 @@ import CLC.Stackage.Runner.Env buildEnv, cache, completePackageSet, + noCabalUpdate, noCache, noCleanup, retryFailures, @@ -50,6 +51,7 @@ mkRunnerEnv = do { buildEnv, cache = Nothing, completePackageSet = NE.toList buildEnv.packagesToBuild, + noCabalUpdate = True, noCache = False, noCleanup = False, retryFailures = False,