checksrc: avoid extra runs in CI, enable more check locally, fix fallouts
To avoid redundant work in CI and to avoid a single checksrc issue make
all autotools jobs fail. After this patch checksrc issues make fail
the checksrc job, the `dist / verify-out-of-tree-autotools-debug`,
`dist / maketgz-and-verify-in-tree` jobs and the fuzzer job (if run).
Of these, the `dist` jobs replicate local builds, also testing the build
logic.
Also add a script to check the complete local repository, optionally
with the build tree to verify generated C files.
Also:
- automatically run checksrc in subdirectories having a `checksrc`
target. (examples, OS400, tests http/client, unit and tunit)
- tests/libtest: make sure to run `checksrc` on generated `lib1521.c`.
(requires in-tree autotools build.)
- tests: run `checksrc` on targets also for non-`DEBUGBUILD`
builds. It ensures to check `lib1521.c` in CI via job
`dist / maketgz-and-verify-in-tree`.
- src: drop redundant `$(builddir)` in autotools builds.
- scripts: add `checksrc-all.sh` script to check all C sources and
the build directory as an option.
- use the above from CI, also make it verify all generated sources.
- silence `checksrc` issues in generated C sources.
- checksrc: add `-v` option to enable verbose mode.
- checksrc: make verbose mode show checked filename and fix to only
return error on failure.
- make sure that generated C files pass `checksrc`.
Assisted-by: Daniel Stenberg
Closes #17376
diff --git a/.github/workflows/checksrc.yml b/.github/workflows/checksrc.yml
index 4083cc3..990551a 100644
--- a/.github/workflows/checksrc.yml
+++ b/.github/workflows/checksrc.yml
@@ -41,7 +41,7 @@
name: checkout
- name: check
- run: git ls-files -z "*.[ch]" | xargs -0 -n1 ./scripts/checksrc.pl
+ run: scripts/checksrc-all.sh
codespell-cmakelint-pytype-ruff:
runs-on: ubuntu-latest
diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml
index 20d4a7a..c851de0 100644
--- a/.github/workflows/distcheck.yml
+++ b/.github/workflows/distcheck.yml
@@ -109,6 +109,8 @@
make
make test-ci
make install
+ popd
+ scripts/checksrc-all.sh
verify-out-of-tree-cmake:
runs-on: ubuntu-latest
diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml
index 80cae92..fd1ecbc 100644
--- a/.github/workflows/http3-linux.yml
+++ b/.github/workflows/http3-linux.yml
@@ -38,6 +38,7 @@
env:
MAKEFLAGS: -j 5
+ CURL_CI: github
# handled in renovate.json
openssl-version: 3.5.0
# handled in renovate.json
@@ -527,7 +528,6 @@
- name: 'run pytest event based'
env:
CURL_TEST_EVENT: 1
- CURL_CI: github
PYTEST_ADDOPTS: '--color=yes'
PYTEST_XDIST_AUTO_NUM_WORKERS: 4
run: |
diff --git a/.github/workflows/linux-old.yml b/.github/workflows/linux-old.yml
index 8ad3bc0..6d7c6a8 100644
--- a/.github/workflows/linux-old.yml
+++ b/.github/workflows/linux-old.yml
@@ -47,6 +47,7 @@
env:
MAKEFLAGS: -j 5
+ CURL_CI: github
DEBIAN_FRONTEND: noninteractive
jobs:
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 52328ba..295d8a6 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -37,6 +37,7 @@
env:
MAKEFLAGS: -j 5
+ CURL_CI: github
CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized'
# unhandled
bearssl-version: 0.6
@@ -698,7 +699,6 @@
- name: 'run pytest'
if: contains(matrix.build.install_steps, 'pytest')
env:
- CURL_CI: github
PYTEST_ADDOPTS: '--color=yes'
PYTEST_XDIST_AUTO_NUM_WORKERS: 4
run: |
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index 0f619ce..c4e558b 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -47,6 +47,7 @@
# newer than the 10.8 required by `CFURLCreateDataAndPropertiesFromResource`.
env:
+ CURL_CI: github
MAKEFLAGS: -j 4
LDFLAGS: -w # suppress 'object file was built for newer macOS version than being linked' warnings
@@ -338,7 +339,6 @@
- name: 'run pytest'
if: ${{ !matrix.build.clang-tidy && contains(matrix.build.install_steps, 'pytest') }}
env:
- CURL_CI: github
PYTEST_ADDOPTS: '--color=yes'
PYTEST_XDIST_AUTO_NUM_WORKERS: 4
run: |
diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml
index 64dde73..eb49a87 100644
--- a/.github/workflows/non-native.yml
+++ b/.github/workflows/non-native.yml
@@ -35,6 +35,9 @@
permissions: {}
+env:
+ CURL_CI: github
+
jobs:
netbsd:
name: 'NetBSD, CM clang openssl ${{ matrix.arch }}'
@@ -141,6 +144,7 @@
architecture: ${{ matrix.arch }}
run: |
export MAKEFLAGS=-j3
+ export CURL_CI=github
# https://ports.freebsd.org/
time sudo pkg install -y autoconf automake libtool \
pkgconf brotli openldap26-client libidn2 libnghttp2 stunnel py311-impacket
@@ -229,6 +233,7 @@
set -e
ln -s /usr/bin/gcpp /usr/bin/cpp # Some tests expect `cpp`, which is named `gcpp` in this env.
export MAKEFLAGS=-j3
+ export CURL_CI=github
time autoreconf -fi
mkdir bld && cd bld && time ../configure --enable-unity --enable-test-bundles --enable-debug --enable-warnings --enable-werror \
--prefix="${HOME}"/install \
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 0a8b813..f4b2ad5 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -35,6 +35,9 @@
permissions: {}
+env:
+ CURL_CI: github
+
jobs:
cygwin:
name: "cygwin, ${{ matrix.build == 'cmake' && 'CM' || 'AM' }} ${{ matrix.platform }} ${{ matrix.name }}"
diff --git a/configure.ac b/configure.ac
index dd88ea5..8a91594 100644
--- a/configure.ac
+++ b/configure.ac
@@ -39,6 +39,7 @@
AC_CONFIG_SRCDIR([lib/urldata.h])
AC_CONFIG_HEADERS(lib/curl_config.h)
+AH_TOP([/* !checksrc! disable COPYRIGHT all */])
AC_CONFIG_MACRO_DIR([m4])
AM_MAINTAINER_MODE
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
@@ -453,6 +454,8 @@
LT_LANG([Windows Resource])
+AM_CONDITIONAL(NOT_CURL_CI, test -z "$CURL_CI")
+
#
# Automake conditionals based on libtool related checks
#
diff --git a/docs/cmdline-opts/pinnedpubkey.md b/docs/cmdline-opts/pinnedpubkey.md
index d21a18f..fab4345 100644
--- a/docs/cmdline-opts/pinnedpubkey.md
+++ b/docs/cmdline-opts/pinnedpubkey.md
@@ -32,8 +32,9 @@
PEM/DER support:
-OpenSSL and GnuTLS (added in 7.39.0), wolfSSL (added in 7.43.0), mbedTLS
-(added in 7.47.0), Secure Transport macOS 10.7+/iOS 10+ (added in 7.54.1),
+OpenSSL and GnuTLS (added in 7.39.0), wolfSSL (added in 7.43.0),
+mbedTLS (added in 7.47.0),
+Secure Transport macOS 10.7+/iOS 10+ (added in 7.54.1),
Schannel (added in 7.58.1)
sha256 support:
diff --git a/docs/examples/Makefile.am b/docs/examples/Makefile.am
index 02ba20b..042922f 100644
--- a/docs/examples/Makefile.am
+++ b/docs/examples/Makefile.am
@@ -69,3 +69,10 @@
checksrc:
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) $(srcdir)/*.c)
+
+if NOT_CURL_CI
+if DEBUGBUILD
+# for debug builds, we scan the sources on all regular make invokes
+all-local: checksrc
+endif
+endif
diff --git a/include/curl/Makefile.am b/include/curl/Makefile.am
index 8864d60..f664f6a 100644
--- a/include/curl/Makefile.am
+++ b/include/curl/Makefile.am
@@ -35,7 +35,9 @@
checksrc:
$(CHECKSRC)@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(top_srcdir)/include/curl $(pkginclude_HEADERS)
+if NOT_CURL_CI
if DEBUGBUILD
# for debug builds, we scan the sources on all regular make invokes
all-local: checksrc
endif
+endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 9c26d4d..8dac236 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -163,10 +163,12 @@
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) \
$(CSOURCES) $(HHEADERS))
+if NOT_CURL_CI
if DEBUGBUILD
# for debug builds, we scan the sources on all regular make invokes
all-local: checksrc
endif
+endif
# disable the tests that are mostly causing false positives
TIDYFLAGS := -checks=-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling -quiet
diff --git a/packages/Makefile.am b/packages/Makefile.am
index 96f52bd..43e544a 100644
--- a/packages/Makefile.am
+++ b/packages/Makefile.am
@@ -50,3 +50,10 @@
checksrc:
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) $(srcdir)/OS400/*.[ch])
+
+if NOT_CURL_CI
+if DEBUGBUILD
+# for debug builds, we scan the sources on all regular make invokes
+all-local: checksrc
+endif
+endif
diff --git a/scripts/Makefile.am b/scripts/Makefile.am
index 1ad250a..03eb804 100644
--- a/scripts/Makefile.am
+++ b/scripts/Makefile.am
@@ -22,9 +22,9 @@
#
###########################################################################
-EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl \
- mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \
- dmaketgz maketgz release-tools.sh verify-release cmakelint.sh mdlinkcheck \
+EXTRA_DIST = coverage.sh completion.pl firefox-db2pem.sh checksrc.pl checksrc-all.sh \
+ mk-ca-bundle.pl mk-unity.pl schemetable.c cd2nroff nroff2cd cdall cd2cd managen \
+ dmaketgz maketgz release-tools.sh verify-release cmakelint.sh mdlinkcheck \
CMakeLists.txt pythonlint.sh randdisable wcurl
dist_bin_SCRIPTS = wcurl
diff --git a/scripts/checksrc-all.sh b/scripts/checksrc-all.sh
new file mode 100755
index 0000000..2c4b3c5
--- /dev/null
+++ b/scripts/checksrc-all.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright (C) Viktor Szakats
+#
+# SPDX-License-Identifier: curl
+
+set -eu
+
+anyfailed=0
+
+for dir in $({
+ if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+ git ls-files '*.[ch]'
+ else
+ find . -name '*.[ch]'
+ fi
+ [ -n "${1:-}" ] && find "$@" -name '*.[ch]'
+ } | grep -v -F '/CMakeFiles/' | sed -E 's|/[^/]+$||' | sort -u); do
+ if ! ./scripts/checksrc.pl "${dir}"/*.[ch]; then
+ anyfailed=1
+ fi
+done
+
+exit "${anyfailed}"
diff --git a/scripts/checksrc.pl b/scripts/checksrc.pl
index 68189b3..f62962f 100755
--- a/scripts/checksrc.pl
+++ b/scripts/checksrc.pl
@@ -39,7 +39,7 @@
my $wlist="";
my @alist;
my $windows_os = $^O eq 'MSWin32' || $^O eq 'cygwin' || $^O eq 'msys';
-my $verbose;
+my $verbose = 0;
my %skiplist;
my %ignore;
@@ -288,6 +288,11 @@
$file = shift @ARGV;
next;
}
+ elsif($file =~ /^-v/) {
+ $verbose = 1;
+ $file = shift @ARGV;
+ next;
+ }
elsif($file =~ /^(-h|--help)/) {
undef $file;
last;
@@ -307,6 +312,7 @@
print " -W[file] Skip the given file - ignore all its flaws\n";
print " -i<n> Indent spaces. Default: 2\n";
print " -m<n> Maximum line length. Default: 79\n";
+ print " -v Verbose\n";
print "\nDetects and warns for these problems:\n";
my @allw = keys %warnings;
push @allw, keys %warnings_extended;
@@ -448,6 +454,11 @@
my $l = "";
my $prep = 0;
my $prevp = 0;
+
+ if($verbose) {
+ printf "Checking file: $file\n";
+ }
+
open(my $R, '<', $file) || die "failed to open $file";
my $incomment=0;
@@ -1123,5 +1134,7 @@
$serrors,
$swarnings;
}
- exit 5; # return failure
+ if($errors || $warnings) {
+ exit 5; # return failure
+ }
}
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 469df6d..9cae87b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -33,6 +33,9 @@
add_custom_command(
OUTPUT "tool_hugehelp.c"
COMMAND ${CMAKE_COMMAND} -E echo "#include \"tool_setup.h\"" > "tool_hugehelp.c"
+ COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable COPYRIGHT all */" >> "tool_hugehelp.c"
+ COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable INCLUDEDUP all */" >> "tool_hugehelp.c"
+ COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable LONGLINE all */" >> "tool_hugehelp.c"
COMMAND ${CMAKE_COMMAND} -E echo "#ifndef HAVE_LIBZ" >> "tool_hugehelp.c"
COMMAND "${PERL_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/mkhelp.pl" < "${CURL_ASCIIPAGE}" >> "tool_hugehelp.c"
COMMAND ${CMAKE_COMMAND} -E echo "#else" >> "tool_hugehelp.c"
diff --git a/src/Makefile.am b/src/Makefile.am
index 5e84a20..2df95a0 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -125,7 +125,7 @@
# Use absolute directory to disable VPATH
ASCIIPAGE=$(top_builddir)/docs/cmdline-opts/curl.txt
MKHELP=$(top_srcdir)/src/mkhelp.pl
-HUGE=$(builddir)/tool_hugehelp.c
+HUGE=tool_hugehelp.c
HUGECMD = $(HUGEIT_$(V))
HUGEIT_0 = @echo " HUGE " $@;
@@ -145,22 +145,29 @@
# This generates the tool_hugehelp.c file in both uncompressed and
# compressed formats.
$(HUGE): $(ASCIIPAGE) $(MKHELP)
- $(HUGECMD) (echo '#include "tool_setup.h"' > $(HUGE); \
- echo '#ifndef HAVE_LIBZ' >> $(HUGE); \
- $(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE); \
- echo '#else' >> $(HUGE); \
- $(PERL) $(MKHELP) -c < $(ASCIIPAGE) >> $(HUGE); \
+ $(HUGECMD)( \
+ echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE); \
+ echo '/* !checksrc! disable INCLUDEDUP all */' >> $(HUGE); \
+ echo '/* !checksrc! disable LONGLINE all */' >> $(HUGE); \
+ echo '#include "tool_setup.h"' >> $(HUGE); \
+ echo '#ifndef HAVE_LIBZ' >> $(HUGE); \
+ $(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE); \
+ echo '#else' >> $(HUGE); \
+ $(PERL) $(MKHELP) -c < $(ASCIIPAGE) >> $(HUGE); \
echo '#endif /* HAVE_LIBZ */' >> $(HUGE) )
else # HAVE_LIBZ
# This generates the tool_hugehelp.c file uncompressed only
$(HUGE): $(ASCIIPAGE) $(MKHELP)
- $(HUGECMD)(echo '#include "tool_setup.h"' > $(HUGE); \
+ $(HUGECMD)( \
+ echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE); \
+ echo '#include "tool_setup.h"' >> $(HUGE); \
$(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE) )
endif
else # USE_MANUAL
# built-in manual has been disabled, make a blank file
$(HUGE):
+ echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE); \
echo '#include "tool_hugehelp.h"' >> $(HUGE)
endif
@@ -168,7 +175,7 @@
curl_hfiles_gen += tool_hugehelp.h
CLEANFILES += $(HUGE)
-CA_EMBED_CSOURCE = $(builddir)/tool_ca_embed.c
+CA_EMBED_CSOURCE = tool_ca_embed.c
curl_cfiles_gen += $(CA_EMBED_CSOURCE)
CLEANFILES += $(CA_EMBED_CSOURCE)
if CURL_CA_EMBED_SET
@@ -178,7 +185,8 @@
$(PERL) $(MK_FILE_EMBED) --var curl_ca_embed < $(CURL_CA_EMBED) > $(CA_EMBED_CSOURCE)
else
$(CA_EMBED_CSOURCE):
- echo 'extern const void *curl_ca_embed; const void *curl_ca_embed;' > $(CA_EMBED_CSOURCE)
+ echo '/* !checksrc! disable COPYRIGHT all */' > $(CA_EMBED_CSOURCE)
+ echo 'extern const void *curl_ca_embed; const void *curl_ca_embed;' >> $(CA_EMBED_CSOURCE)
endif
CHECKSRC = $(CS_$(V))
@@ -190,10 +198,12 @@
checksrc:
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) $(CURL_CFILES) $(CURL_HFILES))
+if NOT_CURL_CI
if DEBUGBUILD
# for debug builds, we scan the sources on all regular make invokes
all-local: checksrc
endif
+endif
# disable the tests that are mostly causing false positives
TIDYFLAGS := -checks=-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling -quiet
diff --git a/src/mk-file-embed.pl b/src/mk-file-embed.pl
index e4dbe35..9daa519 100755
--- a/src/mk-file-embed.pl
+++ b/src/mk-file-embed.pl
@@ -35,6 +35,7 @@
/*
* NEVER EVER edit this manually, fix the mk-file-embed.pl script instead!
*/
+/* !checksrc! disable COPYRIGHT all */
#ifndef CURL_DECLARED_${varname_upper}
#define CURL_DECLARED_${varname_upper}
extern const unsigned char ${varname}[];
diff --git a/tests/http/clients/Makefile.am b/tests/http/clients/Makefile.am
index bae747f..d1acb95 100644
--- a/tests/http/clients/Makefile.am
+++ b/tests/http/clients/Makefile.am
@@ -70,3 +70,7 @@
checksrc:
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) $(srcdir)/*.c)
+
+if NOT_CURL_CI
+all-local: checksrc
+endif
diff --git a/tests/libtest/Makefile.am b/tests/libtest/Makefile.am
index 497493c..07980f8 100644
--- a/tests/libtest/Makefile.am
+++ b/tests/libtest/Makefile.am
@@ -127,13 +127,12 @@
CS_ = $(CS_0)
# ignore generated C files since they play by slightly different rules!
-checksrc:
+checksrc: lib1521.c
$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) \
-W$(srcdir)/libtest_bundle.c \
$(srcdir)/*.[ch])
-if DEBUGBUILD
-# for debug builds, we scan the sources on all regular make invokes
+if NOT_CURL_CI
all-local: checksrc
endif
diff --git a/tests/mk-bundle.pl b/tests/mk-bundle.pl
index 6ca5139..982de70 100755
--- a/tests/mk-bundle.pl
+++ b/tests/mk-bundle.pl
@@ -39,6 +39,7 @@
print <<HEADER
/* !checksrc! disable COPYRIGHT all */
/* !checksrc! disable INCLUDEDUP all */
+/* !checksrc! disable UNUSEDIGNORE all */
#define CURLTESTS_BUNDLED
#define CURLTESTS_BUNDLED_TEST_H
diff --git a/tests/server/Makefile.am b/tests/server/Makefile.am
index 3b3e6a8..8298119 100644
--- a/tests/server/Makefile.am
+++ b/tests/server/Makefile.am
@@ -95,7 +95,6 @@
-W$(srcdir)/server_bundle.c \
$(srcdir)/*.[ch])
-if DEBUGBUILD
-# for debug builds, we scan the sources on all regular make invokes
+if NOT_CURL_CI
all-local: checksrc
endif
diff --git a/tests/tunit/Makefile.am b/tests/tunit/Makefile.am
index 92e160b..01d2767 100644
--- a/tests/tunit/Makefile.am
+++ b/tests/tunit/Makefile.am
@@ -90,5 +90,9 @@
-W$(srcdir)/tool_bundle.c \
$(srcdir)/*.[ch])
+if NOT_CURL_CI
+all-local: checksrc
+endif
+
clean-local:
rm -f $(BUNDLE)
diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am
index 9cf63ac..331d0a9 100644
--- a/tests/unit/Makefile.am
+++ b/tests/unit/Makefile.am
@@ -88,5 +88,9 @@
-W$(srcdir)/unit_bundle.c \
$(srcdir)/*.[ch])
+if NOT_CURL_CI
+all-local: checksrc
+endif
+
clean-local:
rm -f $(BUNDLE)
