Merge remote-tracking branch 'Dragnalith/dynamic_outputs' into dynami…

…c_outputs

.clang-tidy

@@ -1,13 +1,17 @@
 ---
 Checks: '
   ,readability-avoid-const-params-in-decls,
+  ,readability-inconsistent-declaration-parameter-name,
   ,readability-non-const-parameter,
   ,readability-redundant-string-cstr,
   ,readability-redundant-string-init,
+  ,readability-simplify-boolean-expr,
 '
 WarningsAsErrors: '
   ,readability-avoid-const-params-in-decls,
+  ,readability-inconsistent-declaration-parameter-name,
   ,readability-non-const-parameter,
   ,readability-redundant-string-cstr,
   ,readability-redundant-string-init,
+  ,readability-simplify-boolean-expr,
 '

.github/workflows/linux.yml

@@ -123,3 +123,24 @@ jobs:
     - name: clang-tidy
       run: /usr/lib/llvm-10/share/clang/run-clang-tidy.py -header-filter=src
       working-directory: build-clang
+
+  build-with-python:
+    runs-on: [ubuntu-latest]
+    container:
+      image: ${{ matrix.image }}
+    strategy:
+      matrix:
+        image: ['ubuntu:14.04', 'ubuntu:16.04', 'ubuntu:18.04']
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        apt update
+        apt install -y g++ python3
+    - name: ${{ matrix.image }}
+      run: |
+        python3 configure.py --bootstrap
+        ./ninja all
+        ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots
+        python3 misc/ninja_syntax_test.py
+        ./misc/output_test.py

.github/workflows/macos.yml

@@ -21,12 +21,11 @@ jobs:
       env:
         MACOSX_DEPLOYMENT_TARGET: 10.12
       run: |
-        sudo xcode-select -s /Applications/Xcode_12.2.app
         cmake -Bbuild -GXcode '-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64'
         cmake --build build --config Release
 
     - name: Test ninja
-      run: ctest -vv
+      run: ctest -C Release -vv
       working-directory: build
 
     - name: Create ninja archive

.github/workflows/windows.yml

@@ -19,10 +19,15 @@ jobs:
     - name: Build ninja
       shell: bash
       run: |
-        cmake -DCMAKE_BUILD_TYPE=Release -B build
+        cmake -Bbuild
+        cmake --build build --parallel --config Debug
         cmake --build build --parallel --config Release
 
-    - name: Test ninja
+    - name: Test ninja (Debug)
+      run: .\ninja_test.exe
+      working-directory: build/Debug
+
+    - name: Test ninja (Release)
       run: .\ninja_test.exe
       working-directory: build/Release
 

.gitignore

@@ -38,3 +38,8 @@
 
 # Qt Creator project files
 /CMakeLists.txt.user
+
+# clangd
+/.clangd/
+/compile_commands.json
+/.cache/

CMakeLists.txt

@@ -18,16 +18,18 @@ endif()
 # --- compiler flags
 if(MSVC)
 	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
-	string(APPEND CMAKE_CXX_FLAGS " /W4 /GR- /Zc:__cplusplus")
+	string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+	add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus)
+	add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
 else()
 	include(CheckCXXCompilerFlag)
 	check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated)
 	if(flag_no_deprecated)
-		string(APPEND CMAKE_CXX_FLAGS " -Wno-deprecated")
+		add_compile_options(-Wno-deprecated)
 	endif()
 	check_cxx_compiler_flag(-fdiagnostics-color flag_color_diag)
 	if(flag_color_diag)
-		string(APPEND CMAKE_CXX_FLAGS " -fdiagnostics-color")
+		add_compile_options(-fdiagnostics-color)
 	endif()
 endif()
 
@@ -53,22 +55,30 @@ target_include_directories(libninja-re2c PRIVATE src)
 function(check_platform_supports_browse_mode RESULT)
 	# Make sure the inline.sh script works on this platform.
 	# It uses the shell commands such as 'od', which may not be available.
+
 	execute_process(
 		COMMAND sh -c "echo 'TEST' | src/inline.sh var"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 		RESULT_VARIABLE inline_result
 		OUTPUT_QUIET
 		ERROR_QUIET
 	)
 	if(NOT inline_result EQUAL "0")
 		# The inline script failed, so browse mode is not supported.
 		set(${RESULT} "0" PARENT_SCOPE)
+		if(NOT WIN32)
+			message(WARNING "browse feature omitted due to inline script failure")
+		endif()
 		return()
 	endif()
 
 	# Now check availability of the unistd header
 	check_include_file_cxx(unistd.h PLATFORM_HAS_UNISTD_HEADER)
 	set(${RESULT} "${PLATFORM_HAS_UNISTD_HEADER}" PARENT_SCOPE)
+	if(NOT PLATFORM_HAS_UNISTD_HEADER)
+		message(WARNING "browse feature omitted due to missing unistd.h")
+	endif()
+
 endfunction()
 
 check_platform_supports_browse_mode(platform_supports_ninja_browse)
@@ -81,6 +91,7 @@ add_library(libninja OBJECT
 	src/clparser.cc
 	src/dyndep.cc
 	src/dyndep_parser.cc
+	src/dynout_parser.cc
 	src/debug_flags.cc
 	src/deps_log.cc
 	src/disk_interface.cc
@@ -91,8 +102,10 @@ add_library(libninja OBJECT
 	src/line_printer.cc
 	src/manifest_parser.cc
 	src/metrics.cc
+	src/missing_deps.cc
 	src/parser.cc
 	src/state.cc
+	src/status.cc
 	src/string_piece_util.cc
 	src/util.cc
 	src/version.cc
@@ -128,13 +141,17 @@ endif()
 # On IBM i (identified as "OS400" for compatibility reasons) and AIX, this fixes missing
 # PRId64 (and others) at compile time in C++ sources
 if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
-	string(APPEND CMAKE_CXX_FLAGS " -D__STDC_FORMAT_MACROS")
+	add_compile_definitions(__STDC_FORMAT_MACROS)
 endif()
 
 # Main executable is library plus main() function.
 add_executable(ninja src/ninja.cc)
 target_link_libraries(ninja PRIVATE libninja libninja-re2c)
 
+if(WIN32)
+  target_sources(ninja PRIVATE windows/ninja.manifest)
+endif()
+
 # Adds browse mode into the ninja binary if it's supported by the host platform.
 if(platform_supports_ninja_browse)
 	# Inlines src/browse.py into the browse_py.h header, so that it can be included
@@ -143,20 +160,20 @@ if(platform_supports_ninja_browse)
 		OUTPUT build/browse_py.h
 		MAIN_DEPENDENCY src/browse.py
 		DEPENDS src/inline.sh
-		COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/build
+		COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/build
 		COMMAND src/inline.sh kBrowsePy
 						< src/browse.py
-						> ${CMAKE_BINARY_DIR}/build/browse_py.h
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+						> ${PROJECT_BINARY_DIR}/build/browse_py.h
+		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 		VERBATIM
 	)
 
 	target_compile_definitions(ninja PRIVATE NINJA_HAVE_BROWSE)
 	target_sources(ninja PRIVATE src/browse.cc)
 	set_source_files_properties(src/browse.cc
 		PROPERTIES
-			OBJECT_DEPENDS "${CMAKE_BINARY_DIR}/build/browse_py.h"
-			INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}"
+			OBJECT_DEPENDS "${PROJECT_BINARY_DIR}/build/browse_py.h"
+			INCLUDE_DIRECTORIES "${PROJECT_BINARY_DIR}"
 			COMPILE_DEFINITIONS NINJA_PYTHON="python"
 	)
 endif()
@@ -177,6 +194,7 @@ if(BUILD_TESTING)
     src/graph_test.cc
     src/lexer_test.cc
     src/manifest_parser_test.cc
+    src/missing_deps_test.cc
     src/ninja_test.cc
     src/state_test.cc
     src/string_piece_util_test.cc
@@ -203,11 +221,11 @@ if(BUILD_TESTING)
 
   if(CMAKE_SYSTEM_NAME STREQUAL "AIX" AND CMAKE_SIZEOF_VOID_P EQUAL 4)
     # These tests require more memory than will fit in the standard AIX shared stack/heap (256M)
-    target_link_libraries(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000")
-    target_link_libraries(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000")
+    target_link_options(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000")
+    target_link_options(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000")
   endif()
 
-  add_test(NinjaTest ninja_test)
+  add_test(NAME NinjaTest COMMAND ninja_test)
 endif()
 
 install(TARGETS ninja DESTINATION bin)

configure.py

@@ -503,6 +503,7 @@ def has_re2c():
              'disk_interface',
              'dyndep',
              'dyndep_parser',
+             'dynout_parser',
              'edit_distance',
              'eval_env',
              'graph',
@@ -511,8 +512,10 @@ def has_re2c():
              'line_printer',
              'manifest_parser',
              'metrics',
+             'missing_deps',
              'parser',
              'state',
+             'status',
              'string_piece_util',
              'util',
              'version']:
@@ -577,8 +580,10 @@ def has_re2c():
              'graph_test',
              'lexer_test',
              'manifest_parser_test',
+             'missing_deps_test',
              'ninja_test',
              'state_test',
+             'status_test',
              'string_piece_util_test',
              'subprocess_test',
              'test',

doc/manual.asciidoc

@@ -285,6 +285,21 @@ _Available since Ninja 1.2._
 `deps`:: show all dependencies stored in the `.ninja_deps` file. When given a
 target, show just the target's dependencies. _Available since Ninja 1.4._
 
+`missingdeps`:: given a list of targets, look for targets that depend on
+a generated file, but do not have a properly (possibly transitive) dependency
+on the generator.  Such targets may cause build flakiness on clean builds.
++
+The broken targets can be found assuming deps log / depfile dependency
+information is correct.  Any target that depends on a generated file (output
+of a generator-target) implicitly, but does not have an explicit or order-only
+dependency path to the generator-target, is considered broken.
++
+The tool's findings can be verified by trying to build the listed targets in
+a clean outdir without buidling any other targets.  The build should fail for
+each of them with a missing include error or equivalent pointing to the
+generated file.
+_Available since Ninja 1.11._
+
 `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._
 
 `restat`:: updates all recorded file modification timestamps in the `.ninja_log`
@@ -294,6 +309,22 @@ file. _Available since Ninja 1.10._
 if they have one).  It can be used to know which rule name to pass to
 +ninja -t targets rule _name_+ or +ninja -t compdb+.
 
+`wincodepage`:: Available on Windows hosts (_since Ninja 1.11_).
+Prints the Windows code page whose encoding is expected in the build file.
+The output has the form:
++
+----
+Build file encoding: <codepage>
+----
++
+Additional lines may be added in future versions of Ninja.
++
+The `<codepage>` is one of:
+
+`UTF-8`::: Encode as UTF-8.
+
+`ANSI`::: Encode to the system-wide ANSI code page.
+
 Writing your own Ninja files
 ----------------------------
 
@@ -830,6 +861,12 @@ keys.
    stored as `.ninja_deps` in the `builddir`, see <<ref_toplevel,the
    discussion of `builddir`>>.
 
+`dynout`:: path to an optional _dynout file_ that contains the list
+  of outputs generated by the rule. The dynout file syntax except one
+  path per line. This is to make Ninja aware of dynamic outputs, so 
+  the command is re-run if the some dynamic outputs are missing, and 
+  dynamic outputs are clean when using `-t clean` tool.
+
 `msvc_deps_prefix`:: _(Available since Ninja 1.5.)_ defines the string
   which should be stripped from msvc's /showIncludes output. Only
   needed when `deps = msvc` and no English Visual Studio version is used.
@@ -961,8 +998,9 @@ source file of a compile command.
 +
 This is for expressing dependencies that don't show up on the
 command line of the command; for example, for a rule that runs a
-script, the script itself should be an implicit dependency, as
-changes to the script should cause the output to rebuild.
+script that reads a hardcoded file, the hardcoded file should 
+be an implicit dependency, as changes to the file should cause 
+the output to rebuild, even though it doesn't show up in the arguments.
 +
 Note that dependencies as loaded through depfiles have slightly different
 semantics, as described in the <<ref_rule,rule reference>>.