diff --git a/Makefile b/Makefile index 490a1ad5..7950cf93 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ sandbox-index: compile $(PREFIX)/bin/sourcemeta-one-index \ $(SANDBOX)/one.json $(OUTPUT)/sandbox --configuration $(PREFIX)/bin/sourcemeta-one-index \ - $(SANDBOX)/one.json $(OUTPUT)/sandbox --profile + $(SANDBOX)/one.json $(OUTPUT)/sandbox --profile --verbose .PHONY: sandbox sandbox: sandbox-index diff --git a/src/index/index.cc b/src/index/index.cc index 3c0a10db..b390fdb1 100644 --- a/src/index/index.cc +++ b/src/index/index.cc @@ -75,16 +75,15 @@ static auto print_progress(std::mutex &mutex, const std::size_t threads, } template -static auto DISPATCH(const std::filesystem::path &destination, - const typename Handler::Context &context, - std::mutex &mutex, const std::string_view title, - const std::string_view prefix, - const std::string_view suffix, - sourcemeta::one::Build &output, const Deps &...deps) - -> bool { +static auto +DISPATCH(const std::filesystem::path &destination, + const typename Handler::Context &context, std::mutex &mutex, + const bool verbose, const std::string_view title, + const std::string_view prefix, const std::string_view suffix, + sourcemeta::one::Build &output, const Deps &...deps) -> bool { const auto was_built{output.dispatch( Handler::handler, destination, context, deps...)}; - if (!was_built) { + if (!was_built && verbose) { std::lock_guard lock{mutex}; std::cerr << "(skip) " << title << ": " << prefix << " [" << suffix << "]\n"; @@ -220,6 +219,7 @@ static auto index_main(const std::string_view &program, const auto concurrency{app.contains("concurrency") ? std::stoull(app.at("concurrency").front().data()) : std::thread::hardware_concurrency()}; + const auto verbose{app.contains("verbose")}; ///////////////////////////////////////////////////////////////////////////// // (8) First pass to locate all of the schemas we will be indexing @@ -274,14 +274,14 @@ static auto index_main(const std::string_view &program, sourcemeta::core::parallel_for_each( detected_schemas.begin(), detected_schemas.end(), [&configuration, &resolver, &mutex, &detected_schemas, - &app](const auto &detected, const auto threads, const auto cursor) { + verbose](const auto &detected, const auto threads, const auto cursor) { print_progress(mutex, threads, "Resolving", detected.path.filename().string(), cursor, detected_schemas.size()); const auto mapping{ resolver.add(configuration.url, detected.collection_relative_path, detected.collection.get(), detected.path)}; - if (app.contains("verbose")) { + if (verbose) { std::lock_guard lock{mutex}; std::cerr << mapping.first.get() << " => " << mapping.second.get() << "\n"; @@ -303,14 +303,14 @@ static auto index_main(const std::string_view &program, sourcemeta::core::parallel_for_each( resolver.begin(), resolver.end(), [&schemas_path, &resolver, &mutex, &output, &mark_configuration_path, - &mark_version_path](const auto &schema, const auto threads, - const auto cursor) { + &mark_version_path, + verbose](const auto &schema, const auto threads, const auto cursor) { print_progress(mutex, threads, "Ingesting", schema.first, cursor, resolver.size()); const auto destination{schemas_path / schema.second.relative_path / SENTINEL / "schema.metapack"}; DISPATCH( - destination, {schema.first, resolver}, mutex, "Ingesting", + destination, {schema.first, resolver}, mutex, verbose, "Ingesting", schema.first, "materialise", output, schema.second.path, mark_configuration_path, mark_version_path); @@ -335,8 +335,8 @@ static auto index_main(const std::string_view &program, sourcemeta::core::parallel_for_each( resolver.begin(), resolver.end(), [&schemas_path, &explorer_path, &resolver, &mutex, &output, - &mark_configuration_path, &mark_version_path]( - const auto &schema, const auto threads, const auto cursor) { + &mark_configuration_path, &mark_version_path, + verbose](const auto &schema, const auto threads, const auto cursor) { print_progress(mutex, threads, "Analysing", schema.first, cursor, resolver.size()); const auto base_path{schemas_path / schema.second.relative_path / @@ -348,49 +348,51 @@ static auto index_main(const std::string_view &program, const auto health_metapack{base_path / "health.metapack"}; DISPATCH( - base_path / "positions.metapack", resolver, mutex, "Analysing", - schema.first, "positions", output, schema_metapack, + base_path / "positions.metapack", resolver, mutex, verbose, + "Analysing", schema.first, "positions", output, schema_metapack, mark_version_path); DISPATCH( - base_path / "locations.metapack", resolver, mutex, "Analysing", - schema.first, "locations", output, schema_metapack, + base_path / "locations.metapack", resolver, mutex, verbose, + "Analysing", schema.first, "locations", output, schema_metapack, mark_version_path); DISPATCH( - dependencies_metapack, resolver, mutex, "Analysing", schema.first, - "dependencies", output, schema_metapack, mark_version_path); + dependencies_metapack, resolver, mutex, verbose, "Analysing", + schema.first, "dependencies", output, schema_metapack, + mark_version_path); DISPATCH( - base_path / "stats.metapack", resolver, mutex, "Analysing", + base_path / "stats.metapack", resolver, mutex, verbose, "Analysing", schema.first, "stats", output, schema_metapack, mark_version_path); DISPATCH( health_metapack, {std::ref(resolver), std::cref(schema.second.collection.get())}, - mutex, "Analysing", schema.first, "health", output, schema_metapack, - dependencies_metapack, mark_version_path); + mutex, verbose, "Analysing", schema.first, "health", output, + schema_metapack, dependencies_metapack, mark_version_path); DISPATCH( - bundle_metapack, resolver, mutex, "Analysing", schema.first, - "bundle", output, schema_metapack, dependencies_metapack, - mark_version_path); + bundle_metapack, resolver, mutex, verbose, "Analysing", + schema.first, "bundle", output, schema_metapack, + dependencies_metapack, mark_version_path); DISPATCH( - base_path / "editor.metapack", resolver, mutex, "Analysing", - schema.first, "editor", output, bundle_metapack, mark_version_path); + base_path / "editor.metapack", resolver, mutex, verbose, + "Analysing", schema.first, "editor", output, bundle_metapack, + mark_version_path); if (attribute_not_disabled(schema.second.collection.get(), "x-sourcemeta-one:evaluate")) { DISPATCH( base_path / "blaze-exhaustive.metapack", - sourcemeta::blaze::Mode::Exhaustive, mutex, "Analysing", + sourcemeta::blaze::Mode::Exhaustive, mutex, verbose, "Analysing", schema.first, "blaze-exhaustive", output, bundle_metapack, mark_version_path); DISPATCH( base_path / "blaze-fast.metapack", - sourcemeta::blaze::Mode::FastValidation, mutex, "Analysing", - schema.first, "blaze-fast", output, bundle_metapack, + sourcemeta::blaze::Mode::FastValidation, mutex, verbose, + "Analysing", schema.first, "blaze-fast", output, bundle_metapack, mark_version_path); } @@ -399,7 +401,7 @@ static auto index_main(const std::string_view &program, "schema.metapack", {resolver, schema.second.collection.get(), schema.second.relative_path}, - mutex, "Analysing", schema.first, "metadata", output, + mutex, verbose, "Analysing", schema.first, "metadata", output, schema_metapack, health_metapack, dependencies_metapack, mark_configuration_path, mark_version_path); }, @@ -552,7 +554,7 @@ static auto index_main(const std::string_view &program, const auto dependency_tree_rebuilt{ DISPATCH( - dependency_tree_path, resolver, mutex, "Reviewing", + dependency_tree_path, resolver, mutex, verbose, "Reviewing", display_schemas_path.string(), "dependencies", output, dependencies)}; // Determine which schemas' dependents actually changed by diffing @@ -612,12 +614,12 @@ static auto index_main(const std::string_view &program, sourcemeta::core::parallel_for_each( rework_entries.begin(), rework_entries.end(), - [&rework_entries, &mutex, &output, &dependency_tree_path]( - const auto &entry, const auto threads, const auto cursor) { + [&rework_entries, &mutex, &output, &dependency_tree_path, + verbose](const auto &entry, const auto threads, const auto cursor) { print_progress(mutex, threads, "Reworking", entry.uri.get(), cursor, rework_entries.size()); DISPATCH( - entry.dependents_path, entry.uri.get(), mutex, "Reworking", + entry.dependents_path, entry.uri.get(), mutex, verbose, "Reworking", entry.uri.get(), "dependents", output, dependency_tree_path); }, concurrency, THREAD_STACK_SIZE); @@ -631,8 +633,8 @@ static auto index_main(const std::string_view &program, print_progress(mutex, concurrency, "Producing", display_explorer_path.string(), 0, 100); DISPATCH( - explorer_path / SENTINEL / "search.metapack", nullptr, mutex, "Producing", - display_explorer_path.string(), "search", output, summaries); + explorer_path / SENTINEL / "search.metapack", nullptr, mutex, verbose, + "Producing", display_explorer_path.string(), "search", output, summaries); PROFILE_END(profiling, "Search"); @@ -659,8 +661,8 @@ static auto index_main(const std::string_view &program, .output = output, .explorer_path = explorer_path, .schemas_path = schemas_path}, - mutex, "Producing", relative_path.string(), "directory", output, - local_summaries); + mutex, verbose, "Producing", relative_path.string(), "directory", + output, local_summaries); local_summaries.pop_back(); local_summaries.pop_back(); } @@ -676,8 +678,8 @@ static auto index_main(const std::string_view &program, directories.begin(), directories.end(), [&configuration, &schemas_path, &explorer_path, &directories, &summaries, &mutex, &output, &mark_configuration_path, - &mark_version_path](const auto &entry, const auto threads, - const auto cursor) { + &mark_version_path, + verbose](const auto &entry, const auto threads, const auto cursor) { const auto relative_path{ std::filesystem::relative(entry, schemas_path)}; print_progress(mutex, threads, "Rendering", relative_path.string(), @@ -688,21 +690,21 @@ static auto index_main(const std::string_view &program, "directory.metapack"}; DISPATCH( explorer_path / SENTINEL / "directory-html.metapack", - configuration, mutex, "Rendering", relative_path.string(), - "index", output, directory_metapack, mark_configuration_path, - mark_version_path); + configuration, mutex, verbose, "Rendering", + relative_path.string(), "index", output, directory_metapack, + mark_configuration_path, mark_version_path); DISPATCH( explorer_path / SENTINEL / "404.metapack", configuration, mutex, - "Rendering", relative_path.string(), "not-found", output, - mark_configuration_path, mark_version_path); + verbose, "Rendering", relative_path.string(), "not-found", + output, mark_configuration_path, mark_version_path); } else { const auto directory_metapack{explorer_path / relative_path / SENTINEL / "directory.metapack"}; DISPATCH( explorer_path / relative_path / SENTINEL / "directory-html.metapack", - configuration, mutex, "Rendering", relative_path.string(), - "directory", output, directory_metapack, + configuration, mutex, verbose, "Rendering", + relative_path.string(), "directory", output, directory_metapack, mark_configuration_path, mark_version_path); } }, @@ -712,8 +714,8 @@ static auto index_main(const std::string_view &program, summaries.begin(), summaries.end(), [&configuration, &schemas_path, &explorer_path, &directories, &summaries, &mutex, &output, &mark_configuration_path, - &mark_version_path](const auto &entry, const auto threads, - const auto cursor) { + &mark_version_path, + verbose](const auto &entry, const auto threads, const auto cursor) { const auto relative_path{ std::filesystem::relative(entry, explorer_path) .parent_path() @@ -729,8 +731,8 @@ static auto index_main(const std::string_view &program, "dependents.metapack"}; DISPATCH( entry.parent_path() / "schema-html.metapack", configuration, - mutex, "Rendering", relative_path.string(), "schema", output, - entry, schema_deps_metapack, schema_health_metapack, + mutex, verbose, "Rendering", relative_path.string(), "schema", + output, entry, schema_deps_metapack, schema_health_metapack, schema_dependents_metapack, mark_configuration_path, mark_version_path); }, @@ -779,8 +781,9 @@ static auto index_main(const std::string_view &program, const auto display_routes_path{ std::filesystem::relative(routes_path, output.path())}; DISPATCH( - routes_path, router, mutex, "Producing", display_routes_path.string(), - "routes", output, mark_configuration_path, mark_version_path); + routes_path, router, mutex, verbose, "Producing", + display_routes_path.string(), "routes", output, mark_configuration_path, + mark_version_path); PROFILE_END(profiling, "Routes"); diff --git a/test/cli/index/common/comment-removed-on-rebuild.sh b/test/cli/index/common/comment-removed-on-rebuild.sh index 6cf850e5..521ce369 100755 --- a/test/cli/index/common/comment-removed-on-rebuild.sh +++ b/test/cli/index/common/comment-removed-on-rebuild.sh @@ -45,7 +45,7 @@ diff "$TMP/output/comment.json" "$TMP/expected_comment.txt" # Second run without --comment -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" diff --git a/test/cli/index/common/comment-updated-on-rebuild.sh b/test/cli/index/common/comment-updated-on-rebuild.sh index dbe41f0d..5543e398 100755 --- a/test/cli/index/common/comment-updated-on-rebuild.sh +++ b/test/cli/index/common/comment-updated-on-rebuild.sh @@ -45,7 +45,7 @@ diff "$TMP/output/comment.json" "$TMP/expected_comment.txt" # Second run with --comment "second" -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --comment "second" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --comment "second" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" diff --git a/test/cli/index/common/rebuild-cache.sh b/test/cli/index/common/rebuild-cache.sh index a09b36ea..bc026187 100755 --- a/test/cli/index/common/rebuild-cache.sh +++ b/test/cli/index/common/rebuild-cache.sh @@ -67,13 +67,14 @@ diff "$TMP/output.txt" "$TMP/expected.txt" # Run it once more -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json Detecting: $(realpath "$TMP")/schemas/foo.json (#1) (100%) Resolving: foo.json +https://example.com/foo => https://sourcemeta.com/example/schemas/foo (100%) Ingesting: https://sourcemeta.com/example/schemas/foo (skip) Ingesting: https://sourcemeta.com/example/schemas/foo [materialise] (100%) Analysing: https://sourcemeta.com/example/schemas/foo @@ -120,13 +121,14 @@ cat << 'EOF' > "$TMP/schemas/foo.json" "type": "string" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json Detecting: $(realpath "$TMP")/schemas/foo.json (#1) (100%) Resolving: foo.json +https://example.com/foo => https://sourcemeta.com/example/schemas/foo (100%) Ingesting: https://sourcemeta.com/example/schemas/foo (100%) Analysing: https://sourcemeta.com/example/schemas/foo ( 33%) Reviewing: schemas @@ -147,13 +149,14 @@ diff "$TMP/output.txt" "$TMP/expected.txt" # Touch an output file (the indexer trusts its own marks, so this is a no-op) touch "$TMP/output/configuration.json" -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json Detecting: $(realpath "$TMP")/schemas/foo.json (#1) (100%) Resolving: foo.json +https://example.com/foo => https://sourcemeta.com/example/schemas/foo (100%) Ingesting: https://sourcemeta.com/example/schemas/foo (skip) Ingesting: https://sourcemeta.com/example/schemas/foo [materialise] (100%) Analysing: https://sourcemeta.com/example/schemas/foo diff --git a/test/cli/index/common/rebuild-modify-cache.sh b/test/cli/index/common/rebuild-modify-cache.sh index fe1716ea..7f7f73f3 100755 --- a/test/cli/index/common/rebuild-modify-cache.sh +++ b/test/cli/index/common/rebuild-modify-cache.sh @@ -55,7 +55,7 @@ EOF "$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 > /dev/null 2>&1 # Run 3: re-index with no changes. Everything should be fully cached. -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" @@ -63,6 +63,7 @@ Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json Detecting: $(realpath "$TMP")/schemas/a.json (#1) (100%) Resolving: a.json +https://example.com/a => https://sourcemeta.com/example/schemas/a (100%) Ingesting: https://sourcemeta.com/example/schemas/a (skip) Ingesting: https://sourcemeta.com/example/schemas/a [materialise] (100%) Analysing: https://sourcemeta.com/example/schemas/a diff --git a/test/cli/index/common/rebuild-nested-directories.sh b/test/cli/index/common/rebuild-nested-directories.sh index 3722ee44..f4f95783 100755 --- a/test/cli/index/common/rebuild-nested-directories.sh +++ b/test/cli/index/common/rebuild-nested-directories.sh @@ -91,7 +91,7 @@ cat << 'EOF' > "$TMP/schemas-left-a/s5.json" "$id": "https://example.com/left/left-a/s5" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" grep '(skip)' "$TMP/output.txt" | LC_ALL=C sort > "$TMP/actual_skips.txt" diff --git a/test/cli/index/common/rebuild-one-to-zero.sh b/test/cli/index/common/rebuild-one-to-zero.sh index 4128fef0..916d6289 100755 --- a/test/cli/index/common/rebuild-one-to-zero.sh +++ b/test/cli/index/common/rebuild-one-to-zero.sh @@ -106,7 +106,7 @@ diff "$TMP/manifest.txt" "$TMP/expected_manifest.txt" # Second run: delete the schema file rm "$TMP/schemas/test.json" -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" diff --git a/test/cli/index/common/rebuild-two-to-three-with-ref.sh b/test/cli/index/common/rebuild-two-to-three-with-ref.sh index 24963de1..ae9be74b 100755 --- a/test/cli/index/common/rebuild-two-to-three-with-ref.sh +++ b/test/cli/index/common/rebuild-two-to-three-with-ref.sh @@ -61,7 +61,7 @@ cat << 'EOF' > "$TMP/schemas/c.json" } } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" grep '(skip)' "$TMP/output.txt" | LC_ALL=C sort > "$TMP/actual_skips.txt" diff --git a/test/cli/index/common/rebuild-two-to-three.sh b/test/cli/index/common/rebuild-two-to-three.sh index d3430f8b..2a5e306d 100755 --- a/test/cli/index/common/rebuild-two-to-three.sh +++ b/test/cli/index/common/rebuild-two-to-three.sh @@ -60,7 +60,7 @@ cat << 'EOF' > "$TMP/schemas/c.json" "$id": "https://example.com/c" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" grep '(skip)' "$TMP/output.txt" | LC_ALL=C sort > "$TMP/actual_skips.txt" @@ -96,7 +96,7 @@ EOF diff "$TMP/actual_skips.txt" "$TMP/expected_skips.txt" # Run 3: re-index with no changes. All three schemas should be fully cached. -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" grep '(skip) Ingesting:' "$TMP/output.txt" | sort > "$TMP/ingest_actual.txt" diff --git a/test/cli/index/common/rebuild-zero-to-one.sh b/test/cli/index/common/rebuild-zero-to-one.sh index e2733814..5abc3dd1 100755 --- a/test/cli/index/common/rebuild-zero-to-one.sh +++ b/test/cli/index/common/rebuild-zero-to-one.sh @@ -74,7 +74,7 @@ cat << 'EOF' > "$TMP/schemas/test.json" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 --verbose 2> "$TMP/output.txt" remove_threads_information "$TMP/output.txt" cat << EOF > "$TMP/expected.txt" @@ -82,6 +82,7 @@ Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json Detecting: $(realpath "$TMP")/schemas/test.json (#1) (100%) Resolving: test.json +https://example.com/test => https://sourcemeta.com/schemas/test (100%) Ingesting: https://sourcemeta.com/schemas/test (100%) Analysing: https://sourcemeta.com/schemas/test ( 33%) Reviewing: schemas