[experiments] Re-structure experiments codegen to make it more modular and re-usable (#33263)
diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl
index 6d9663b..0c0b83d 100644
--- a/bazel/experiments.bzl
+++ b/bazel/experiments.bzl
@@ -1,4 +1,4 @@
-# Copyright 2022 gRPC authors.
+# Copyright 2023 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Automatically generated by tools/codegen/core/gen_experiments.py
+# Auto generated by tools/codegen/core/gen_experiments.py
"""Dictionary of tags to experiments so we know when to test different experiments."""
diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc
index b868371..dc48e9b 100644
--- a/src/core/lib/experiments/experiments.cc
+++ b/src/core/lib/experiments/experiments.cc
@@ -1,4 +1,4 @@
-// Copyright 2022 gRPC authors.
+// Copyright 2023 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Automatically generated by tools/codegen/core/gen_experiments.py
+// Auto generated by tools/codegen/core/gen_experiments.py
#include <grpc/support/port_platform.h>
@@ -25,65 +25,67 @@
"would not indicate completion of a read operation until a specified "
"number of bytes have been read over the socket. Buffers are also "
"allocated according to estimated RPC sizes.";
-const char* const additional_constraints_tcp_frame_size_tuning = "";
+const char* const additional_constraints_tcp_frame_size_tuning = "{}";
const char* const description_tcp_rcv_lowat =
"Use SO_RCVLOWAT to avoid wakeups on the read path.";
-const char* const additional_constraints_tcp_rcv_lowat = "";
+const char* const additional_constraints_tcp_rcv_lowat = "{}";
const char* const description_peer_state_based_framing =
"If set, the max sizes of frames sent to lower layers is controlled based "
"on the peer's memory pressure which is reflected in its max http2 frame "
"size.";
-const char* const additional_constraints_peer_state_based_framing = "";
+const char* const additional_constraints_peer_state_based_framing = "{}";
const char* const description_memory_pressure_controller =
"New memory pressure controller";
-const char* const additional_constraints_memory_pressure_controller = "";
+const char* const additional_constraints_memory_pressure_controller = "{}";
const char* const description_unconstrained_max_quota_buffer_size =
"Discard the cap on the max free pool size for one memory allocator";
const char* const additional_constraints_unconstrained_max_quota_buffer_size =
- "";
+ "{}";
const char* const description_event_engine_client =
"Use EventEngine clients instead of iomgr's grpc_tcp_client";
-const char* const additional_constraints_event_engine_client = "";
+const char* const additional_constraints_event_engine_client = "{}";
const char* const description_monitoring_experiment =
"Placeholder experiment to prove/disprove our monitoring is working";
-const char* const additional_constraints_monitoring_experiment = "";
+const char* const additional_constraints_monitoring_experiment = "{}";
const char* const description_promise_based_client_call =
"If set, use the new gRPC promise based call code when it's appropriate "
"(ie when all filters in a stack are promise based)";
-const char* const additional_constraints_promise_based_client_call = "";
+const char* const additional_constraints_promise_based_client_call = "{}";
const char* const description_free_large_allocator =
"If set, return all free bytes from a \042big\042 allocator";
-const char* const additional_constraints_free_large_allocator = "";
+const char* const additional_constraints_free_large_allocator = "{}";
const char* const description_promise_based_server_call =
"If set, use the new gRPC promise based call code when it's appropriate "
"(ie when all filters in a stack are promise based)";
-const char* const additional_constraints_promise_based_server_call = "";
+const char* const additional_constraints_promise_based_server_call = "{}";
const char* const description_transport_supplies_client_latency =
"If set, use the transport represented value for client latency in "
"opencensus";
-const char* const additional_constraints_transport_supplies_client_latency = "";
+const char* const additional_constraints_transport_supplies_client_latency =
+ "{}";
const char* const description_event_engine_listener =
"Use EventEngine listeners instead of iomgr's grpc_tcp_server";
-const char* const additional_constraints_event_engine_listener = "";
+const char* const additional_constraints_event_engine_listener = "{}";
const char* const description_schedule_cancellation_over_write =
"Allow cancellation op to be scheduled over a write";
-const char* const additional_constraints_schedule_cancellation_over_write = "";
+const char* const additional_constraints_schedule_cancellation_over_write =
+ "{}";
const char* const description_trace_record_callops =
"Enables tracing of call batch initiation and completion.";
-const char* const additional_constraints_trace_record_callops = "";
+const char* const additional_constraints_trace_record_callops = "{}";
const char* const description_event_engine_dns =
"If set, use EventEngine DNSResolver for client channel resolution";
-const char* const additional_constraints_event_engine_dns = "";
+const char* const additional_constraints_event_engine_dns = "{}";
const char* const description_work_stealing =
"If set, use a work stealing thread pool implementation in EventEngine";
-const char* const additional_constraints_work_stealing = "";
+const char* const additional_constraints_work_stealing = "{}";
const char* const description_client_privacy = "If set, client privacy";
-const char* const additional_constraints_client_privacy = "";
+const char* const additional_constraints_client_privacy = "{}";
const char* const description_canary_client_privacy =
"If set, canary client privacy";
-const char* const additional_constraints_canary_client_privacy = "";
+const char* const additional_constraints_canary_client_privacy = "{}";
const char* const description_server_privacy = "If set, server privacy";
-const char* const additional_constraints_server_privacy = "";
+const char* const additional_constraints_server_privacy = "{}";
} // namespace
namespace grpc_core {
diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h
index e43e20c..ced23dd 100644
--- a/src/core/lib/experiments/experiments.h
+++ b/src/core/lib/experiments/experiments.h
@@ -1,4 +1,4 @@
-// Copyright 2022 gRPC authors.
+// Copyright 2023 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Automatically generated by tools/codegen/core/gen_experiments.py
+// Auto generated by tools/codegen/core/gen_experiments.py
//
// This file contains the autogenerated parts of the experiments API.
//
diff --git a/tools/codegen/core/experiments_compiler.py b/tools/codegen/core/experiments_compiler.py
new file mode 100644
index 0000000..8d9a7ae
--- /dev/null
+++ b/tools/codegen/core/experiments_compiler.py
@@ -0,0 +1,396 @@
+#!/usr/bin/env python3
+
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A module to assist in generating experiment related code and artifacts.
+"""
+
+from __future__ import print_function
+
+import collections
+import ctypes
+import datetime
+import json
+import math
+import os
+import re
+import sys
+
+import yaml
+
+_CODEGEN_PLACEHOLDER_TEXT = """
+This file contains the autogenerated parts of the experiments API.
+
+It generates two symbols for each experiment.
+
+For the experiment named new_car_project, it generates:
+
+- a function IsNewCarProjectEnabled() that returns true if the experiment
+ should be enabled at runtime.
+
+- a macro GRPC_EXPERIMENT_IS_INCLUDED_NEW_CAR_PROJECT that is defined if the
+ experiment *could* be enabled at runtime.
+
+The function is used to determine whether to run the experiment or
+non-experiment code path.
+
+If the experiment brings significant bloat, the macro can be used to avoid
+including the experiment code path in the binary for binaries that are size
+sensitive.
+
+By default that includes our iOS and Android builds.
+
+Finally, a small array is included that contains the metadata for each
+experiment.
+
+A macro, GRPC_EXPERIMENTS_ARE_FINAL, controls whether we fix experiment
+configuration at build time (if it's defined) or allow it to be tuned at
+runtime (if it's disabled).
+
+If you are using the Bazel build system, that macro can be configured with
+--define=grpc_experiments_are_final=true
+"""
+
+
+def ToCStr(s, encoding='ascii'):
+ if isinstance(s, str):
+ s = s.encode(encoding)
+ result = ''
+ for c in s:
+ c = chr(c) if isinstance(c, int) else c
+ if not (32 <= ord(c) < 127) or c in ('\\', '"'):
+ result += '\\%03o' % ord(c)
+ else:
+ result += c
+ return '"' + result + '"'
+
+
+def SnakeToPascal(s):
+ return ''.join(x.capitalize() for x in s.split('_'))
+
+
+def PutBanner(files, banner, prefix):
+ # Print a big comment block into a set of files
+ for f in files:
+ for line in banner:
+ if not line:
+ print(prefix, file=f)
+ else:
+ print('%s %s' % (prefix, line), file=f)
+ print(file=f)
+
+
+def PutCopyright(file, prefix):
+ # copy-paste copyright notice from this file
+ with open(__file__) as my_source:
+ copyright = []
+ for line in my_source:
+ if line[0] != '#':
+ break
+ for line in my_source:
+ if line[0] == '#':
+ copyright.append(line)
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ copyright.append(line)
+ PutBanner([file], [line[2:].rstrip() for line in copyright], prefix)
+
+
+class ExperimentDefinition(object):
+
+ def __init__(self, attributes):
+ self._error = False
+ if 'name' not in attributes:
+ print("ERROR: experiment with no name: %r" % attributes)
+ self._error = True
+ if 'description' not in attributes:
+ print("ERROR: no description for experiment %s" %
+ attributes['name'])
+ self._error = True
+ if 'owner' not in attributes:
+ print("ERROR: no owner for experiment %s" % attributes['name'])
+ self._error = True
+ if 'expiry' not in attributes:
+ print("ERROR: no expiry for experiment %s" % attributes['name'])
+ self._error = True
+ if attributes['name'] == 'monitoring_experiment':
+ if attributes['expiry'] != 'never-ever':
+ print("ERROR: monitoring_experiment should never expire")
+ self._error = True
+ if self._error:
+ print("Failed to create experiment definition")
+ return
+ self._allow_in_fuzzing_config = True
+ self._name = attributes['name']
+ self._description = attributes['description']
+ self._expiry = attributes['expiry']
+ self._default = None
+ self._additional_constraints = {}
+ self._test_tags = []
+
+ if 'allow_in_fuzzing_config' in attributes:
+ self._allow_in_fuzzing_config = attributes[
+ 'allow_in_fuzzing_config']
+
+ if 'test_tags' in attributes:
+ self._test_tags = attributes['test_tags']
+
+ def IsValid(self, check_expiry=False):
+ if self._error:
+ return False
+ if not check_expiry:
+ return True
+ if self._name == 'monitoring_experiment' and self._expiry == 'never-ever':
+ return True
+ today = datetime.date.today()
+ two_quarters_from_now = today + datetime.timedelta(days=180)
+ expiry = datetime.datetime.strptime(self._expiry, '%Y/%m/%d').date()
+ if expiry < today:
+ print("ERROR: experiment %s expired on %s" %
+ (self._name, self._expiry))
+ self._error = True
+ if expiry > two_quarters_from_now:
+ print("ERROR: experiment %s expires far in the future on %s" %
+ (self._name, self._expiry))
+ print("expiry should be no more than two quarters from now")
+ self._error = True
+ return not self._error
+
+ def AddRolloutSpecification(self, allowed_defaults, rollout_attributes):
+ if self._error or self._default is not None:
+ return False
+ if rollout_attributes['name'] != self._name:
+ print(
+ "ERROR: Rollout specification does not apply to this experiment: %s"
+ % self._name)
+ return False
+ if 'default' not in rollout_attributes:
+ print("ERROR: no default for experiment %s" %
+ rollout_attributes['name'])
+ self._error = True
+ if rollout_attributes['default'] not in allowed_defaults:
+ print("ERROR: invalid default for experiment %s: %r" %
+ (rollout_attributes['name'], rollout_attributes['default']))
+ self._error = True
+ if 'additional_constraints' in rollout_attributes:
+ self._additional_constraints = rollout_attributes[
+ 'additional_constraints']
+ self._default = rollout_attributes['default']
+ return True
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def description(self):
+ return self._description
+
+ @property
+ def default(self):
+ return self._default
+
+ @property
+ def test_tags(self):
+ return self._test_tags
+
+ @property
+ def allow_in_fuzzing_config(self):
+ return self._allow_in_fuzzing_config
+
+ @property
+ def additional_constraints(self):
+ return self._additional_constraints
+
+
+class ExperimentsCompiler(object):
+
+ def __init__(self,
+ defaults,
+ final_return,
+ final_define,
+ bzl_list_for_defaults=None):
+ self._defaults = defaults
+ self._final_return = final_return
+ self._final_define = final_define
+ self._bzl_list_for_defaults = bzl_list_for_defaults
+ self._experiment_definitions = {}
+ self._experiment_rollouts = {}
+
+ def AddExperimentDefinition(self, experiment_definition):
+ if experiment_definition.name in self._experiment_definitions:
+ print("ERROR: Duplicate experiment definition: %s" %
+ experiment_definition.name)
+ return False
+ self._experiment_definitions[
+ experiment_definition.name] = experiment_definition
+ return True
+
+ def AddRolloutSpecification(self, rollout_attributes):
+ if 'name' not in rollout_attributes:
+ print("ERROR: experiment with no name: %r in rollout_attribute" %
+ rollout_attributes)
+ return False
+ if rollout_attributes['name'] not in self._experiment_definitions:
+ print("WARNING: rollout for an undefined experiment: %s ignored" %
+ rollout_attributes['name'])
+ return (self._experiment_definitions[
+ rollout_attributes['name']].AddRolloutSpecification(
+ self._defaults, rollout_attributes))
+
+ def GenerateExperimentsHdr(self, output_file):
+ with open(output_file, 'w') as H:
+ PutCopyright(H, "//")
+ PutBanner(
+ [H],
+ ["Auto generated by tools/codegen/core/gen_experiments.py"] +
+ _CODEGEN_PLACEHOLDER_TEXT.splitlines(), "//")
+
+ print("#ifndef GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H)
+ print("#define GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H)
+ print(file=H)
+ print("#include <grpc/support/port_platform.h>", file=H)
+ print(file=H)
+ print("#include <stddef.h>", file=H)
+ print("#include \"src/core/lib/experiments/config.h\"", file=H)
+ print(file=H)
+ print("namespace grpc_core {", file=H)
+ print(file=H)
+ print("#ifdef GRPC_EXPERIMENTS_ARE_FINAL", file=H)
+ for _, exp in self._experiment_definitions.items():
+ define_fmt = self._final_define[exp.default]
+ if define_fmt:
+ print(define_fmt %
+ ("GRPC_EXPERIMENT_IS_INCLUDED_%s" % exp.name.upper()),
+ file=H)
+ print(
+ "inline bool Is%sEnabled() { %s }" %
+ (SnakeToPascal(exp.name), self._final_return[exp.default]),
+ file=H)
+ print("#else", file=H)
+ for i, (_, exp) in enumerate(self._experiment_definitions.items()):
+ print("#define GRPC_EXPERIMENT_IS_INCLUDED_%s" %
+ exp.name.upper(),
+ file=H)
+ print(
+ "inline bool Is%sEnabled() { return IsExperimentEnabled(%d); }"
+ % (SnakeToPascal(exp.name), i),
+ file=H)
+ print(file=H)
+ print("constexpr const size_t kNumExperiments = %d;" %
+ len(self._experiment_definitions.keys()),
+ file=H)
+ print(
+ "extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];",
+ file=H)
+ print(file=H)
+ print("#endif", file=H)
+ print("} // namespace grpc_core", file=H)
+ print(file=H)
+ print("#endif // GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H",
+ file=H)
+
+ def GenerateExperimentsSrc(self, output_file):
+ with open(output_file, 'w') as C:
+ PutCopyright(C, "//")
+ PutBanner(
+ [C],
+ ["Auto generated by tools/codegen/core/gen_experiments.py"],
+ "//")
+
+ print("#include <grpc/support/port_platform.h>", file=C)
+ print("#include \"src/core/lib/experiments/experiments.h\"", file=C)
+ print(file=C)
+ print("#ifndef GRPC_EXPERIMENTS_ARE_FINAL", file=C)
+ print("namespace {", file=C)
+ have_defaults = set()
+ for _, exp in self._experiment_definitions.items():
+ print("const char* const description_%s = %s;" %
+ (exp.name, ToCStr(exp.description)),
+ file=C)
+ print(
+ "const char* const additional_constraints_%s = %s;" %
+ (exp.name, ToCStr(json.dumps(exp.additional_constraints))),
+ file=C)
+ have_defaults.add(exp.default)
+ if 'kDefaultForDebugOnly' in have_defaults:
+ print("#ifdef NDEBUG", file=C)
+ if 'kDefaultForDebugOnly' in have_defaults:
+ print("const bool kDefaultForDebugOnly = false;", file=C)
+ print("#else", file=C)
+ if 'kDefaultForDebugOnly' in have_defaults:
+ print("const bool kDefaultForDebugOnly = true;", file=C)
+ print("#endif", file=C)
+ print("}", file=C)
+ print(file=C)
+ print("namespace grpc_core {", file=C)
+ print(file=C)
+ print("const ExperimentMetadata g_experiment_metadata[] = {",
+ file=C)
+ for _, exp in self._experiment_definitions.items():
+ print(
+ " {%s, description_%s, additional_constraints_%s, %s, %s},"
+ % (ToCStr(exp.name), exp.name, exp.name,
+ 'true' if exp.default else 'false',
+ 'true' if exp.allow_in_fuzzing_config else 'false'),
+ file=C)
+ print("};", file=C)
+ print(file=C)
+ print("} // namespace grpc_core", file=C)
+ print("#endif", file=C)
+
+ def GenExperimentsBzl(self, output_file):
+ if self._bzl_list_for_defaults is None:
+ return
+
+ bzl_to_tags_to_experiments = dict(
+ (key, collections.defaultdict(list))
+ for key in self._bzl_list_for_defaults.keys()
+ if key is not None)
+
+ for _, exp in self._experiment_definitions.items():
+ for tag in exp.test_tags:
+ bzl_to_tags_to_experiments[exp.default][tag].append(exp.name)
+
+ with open(output_file, 'w') as B:
+ PutCopyright(B, "#")
+ PutBanner(
+ [B],
+ ["Auto generated by tools/codegen/core/gen_experiments.py"],
+ "#")
+
+ print(
+ "\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"",
+ file=B)
+
+ bzl_to_tags_to_experiments = sorted(
+ (self._bzl_list_for_defaults[default], tags_to_experiments)
+ for default, tags_to_experiments in
+ bzl_to_tags_to_experiments.items()
+ if self._bzl_list_for_defaults[default] is not None)
+
+ print(file=B)
+ print("EXPERIMENTS = {", file=B)
+ for key, tags_to_experiments in bzl_to_tags_to_experiments:
+ print(" \"%s\": {" % key, file=B)
+ for tag, experiments in sorted(tags_to_experiments.items()):
+ print(" \"%s\": [" % tag, file=B)
+ for experiment in sorted(experiments):
+ print(" \"%s\"," % experiment, file=B)
+ print(" ],", file=B)
+ print(" },", file=B)
+ print("}", file=B)
diff --git a/tools/codegen/core/gen_experiments.py b/tools/codegen/core/gen_experiments.py
index 19af812..0dac030 100755
--- a/tools/codegen/core/gen_experiments.py
+++ b/tools/codegen/core/gen_experiments.py
@@ -22,28 +22,12 @@
from __future__ import print_function
-import collections
-import ctypes
-import datetime
-import json
-import math
-import os
-import re
+import argparse
import sys
+import experiments_compiler as exp
import yaml
-# TODO(ctiller): if we ever add another argument switch this to argparse
-check_dates = True
-if sys.argv[1:] == ["--check"]:
- check_dates = False # for formatting checks we don't verify expiry dates
-
-with open('src/core/lib/experiments/experiments.yaml') as f:
- attrs = yaml.safe_load(f.read())
-
-with open('src/core/lib/experiments/rollouts.yaml') as f:
- rollouts = yaml.safe_load(f.read())
-
DEFAULTS = {
'broken': 'false',
False: 'false',
@@ -72,279 +56,80 @@
'debug': 'dbg',
}
-error = False
-today = datetime.date.today()
-two_quarters_from_now = today + datetime.timedelta(days=180)
-experiment_annotation = 'gRPC experiments:'
-for rollout_attr in rollouts:
- if 'name' not in rollout_attr:
- print("experiment with no name: %r" % attr)
- error = True
- continue
- if 'default' not in rollout_attr:
- print("no default for experiment %s" % rollout_attr['name'])
- error = True
- if rollout_attr['default'] not in DEFAULTS:
- print("invalid default for experiment %s: %r" %
- (rollout_attr['name'], rollout_attr['default']))
- error = True
+
+def ParseCommandLineArguments(args):
+ """Wrapper for argparse command line arguments handling.
+
+ Args:
+ args: List of command line arguments.
+
+ Returns:
+ Command line arguments namespace built by argparse.ArgumentParser().
+ """
+ # formatter_class=argparse.ArgumentDefaultsHelpFormatter is not used here
+ # intentionally, We want more formatting than this class can provide.
+ flag_parser = argparse.ArgumentParser()
+ flag_parser.add_argument(
+ '--check',
+ action='store_false',
+ help='If specified, disables checking experiment expiry dates',
+ )
+ flag_parser.add_argument(
+ '--disable_gen_hdrs',
+ action='store_true',
+ help='If specified, disables generation of experiments hdr files',
+ )
+ flag_parser.add_argument(
+ '--disable_gen_srcs',
+ action='store_true',
+ help='If specified, disables generation of experiments source files',
+ )
+ flag_parser.add_argument(
+ '--disable_gen_bzl',
+ action='store_true',
+ help='If specified, disables generation of experiments.bzl file',
+ )
+ return flag_parser.parse_args(args)
+
+
+args = ParseCommandLineArguments(sys.argv[1:])
+
+with open('src/core/lib/experiments/experiments.yaml') as f:
+ attrs = yaml.safe_load(f.read())
+
+with open('src/core/lib/experiments/rollouts.yaml') as f:
+ rollouts = yaml.safe_load(f.read())
+
+compiler = exp.ExperimentsCompiler(DEFAULTS, FINAL_RETURN, FINAL_DEFINE,
+ BZL_LIST_FOR_DEFAULTS)
+
+experiment_annotation = "gRPC Experiments: "
for attr in attrs:
- if 'name' not in attr:
- print("experiment with no name: %r" % attr)
- error = True
- continue # can't run other diagnostics because we don't know a name
- if 'description' not in attr:
- print("no description for experiment %s" % attr['name'])
- error = True
- if 'owner' not in attr:
- print("no owner for experiment %s" % attr['name'])
- error = True
- if 'expiry' not in attr:
- print("no expiry for experiment %s" % attr['name'])
- error = True
- if attr['name'] == 'monitoring_experiment':
- if attr['expiry'] != 'never-ever':
- print("monitoring_experiment should never expire")
- error = True
- else:
- expiry = datetime.datetime.strptime(attr['expiry'], '%Y/%m/%d').date()
- if check_dates:
- if expiry < today:
- print("experiment %s expired on %s" %
- (attr['name'], attr['expiry']))
- error = True
- if expiry > two_quarters_from_now:
- print("experiment %s expires far in the future on %s" %
- (attr['name'], attr['expiry']))
- print("expiry should be no more than two quarters from now")
- error = True
- experiment_annotation += attr['name'] + ':0,'
+ exp_definition = exp.ExperimentDefinition(attr)
+ if not exp_definition.IsValid(args.check):
+ sys.exit(1)
+ experiment_annotation += exp_definition.name + ':0,'
+ if not compiler.AddExperimentDefinition(exp_definition):
+ print("Experiment = %s ERROR adding" % exp_definition.name)
+ sys.exit(1)
if len(experiment_annotation) > 2000:
print("comma-delimited string of experiments is too long")
- error = True
-
-if error:
sys.exit(1)
+for rollout_attr in rollouts:
+ if not compiler.AddRolloutSpecification(rollout_attr):
+ print("ERROR adding rollout spec")
+ sys.exit(1)
-def c_str(s, encoding='ascii'):
- if isinstance(s, str):
- s = s.encode(encoding)
- result = ''
- for c in s:
- c = chr(c) if isinstance(c, int) else c
- if not (32 <= ord(c) < 127) or c in ('\\', '"'):
- result += '\\%03o' % ord(c)
- else:
- result += c
- return '"' + result + '"'
+if not args.disable_gen_hdrs:
+ print("Generating experiments headers")
+ compiler.GenerateExperimentsHdr('src/core/lib/experiments/experiments.h')
+if not args.disable_gen_srcs:
+ print("Generating experiments srcs")
+ compiler.GenerateExperimentsSrc('src/core/lib/experiments/experiments.cc')
-def snake_to_pascal(s):
- return ''.join(x.capitalize() for x in s.split('_'))
-
-
-# utility: print a big comment block into a set of files
-def put_banner(files, banner, prefix):
- for f in files:
- for line in banner:
- if not line:
- print(prefix, file=f)
- else:
- print('%s %s' % (prefix, line), file=f)
- print(file=f)
-
-
-def put_copyright(file, prefix):
- # copy-paste copyright notice from this file
- with open(sys.argv[0]) as my_source:
- copyright = []
- for line in my_source:
- if line[0] != '#':
- break
- for line in my_source:
- if line[0] == '#':
- copyright.append(line)
- break
- for line in my_source:
- if line[0] != '#':
- break
- copyright.append(line)
- put_banner([file], [line[2:].rstrip() for line in copyright], prefix)
-
-
-def get_rollout_attr_for_experiment(name):
- for rollout_attr in rollouts:
- if rollout_attr['name'] == name:
- return rollout_attr
- print('WARNING. experiment: %r has no rollout config. Disabling it.' % name)
- return {'name': name, 'default': 'false'}
-
-
-WTF = """
-This file contains the autogenerated parts of the experiments API.
-
-It generates two symbols for each experiment.
-
-For the experiment named new_car_project, it generates:
-
-- a function IsNewCarProjectEnabled() that returns true if the experiment
- should be enabled at runtime.
-
-- a macro GRPC_EXPERIMENT_IS_INCLUDED_NEW_CAR_PROJECT that is defined if the
- experiment *could* be enabled at runtime.
-
-The function is used to determine whether to run the experiment or
-non-experiment code path.
-
-If the experiment brings significant bloat, the macro can be used to avoid
-including the experiment code path in the binary for binaries that are size
-sensitive.
-
-By default that includes our iOS and Android builds.
-
-Finally, a small array is included that contains the metadata for each
-experiment.
-
-A macro, GRPC_EXPERIMENTS_ARE_FINAL, controls whether we fix experiment
-configuration at build time (if it's defined) or allow it to be tuned at
-runtime (if it's disabled).
-
-If you are using the Bazel build system, that macro can be configured with
---define=grpc_experiments_are_final=true
-"""
-
-with open('src/core/lib/experiments/experiments.h', 'w') as H:
- put_copyright(H, "//")
-
- put_banner(
- [H],
- ["Automatically generated by tools/codegen/core/gen_experiments.py"] +
- WTF.splitlines(), "//")
-
- print("#ifndef GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H)
- print("#define GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H)
- print(file=H)
- print("#include <grpc/support/port_platform.h>", file=H)
- print(file=H)
- print("#include <stddef.h>", file=H)
- print("#include \"src/core/lib/experiments/config.h\"", file=H)
- print(file=H)
- print("namespace grpc_core {", file=H)
- print(file=H)
- print("#ifdef GRPC_EXPERIMENTS_ARE_FINAL", file=H)
- for i, attr in enumerate(attrs):
- rollout_attr = get_rollout_attr_for_experiment(attr['name'])
- define_fmt = FINAL_DEFINE[rollout_attr['default']]
- if define_fmt:
- print(define_fmt %
- ("GRPC_EXPERIMENT_IS_INCLUDED_%s" % attr['name'].upper()),
- file=H)
- print("inline bool Is%sEnabled() { %s }" % (snake_to_pascal(
- attr['name']), FINAL_RETURN[rollout_attr['default']]),
- file=H)
- print("#else", file=H)
- for i, attr in enumerate(attrs):
- print("#define GRPC_EXPERIMENT_IS_INCLUDED_%s" % attr['name'].upper(),
- file=H)
- print("inline bool Is%sEnabled() { return IsExperimentEnabled(%d); }" %
- (snake_to_pascal(attr['name']), i),
- file=H)
- print(file=H)
- print("constexpr const size_t kNumExperiments = %d;" % len(attrs), file=H)
- print(
- "extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];",
- file=H)
- print(file=H)
- print("#endif", file=H)
- print("} // namespace grpc_core", file=H)
- print(file=H)
- print("#endif // GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H)
-
-with open('src/core/lib/experiments/experiments.cc', 'w') as C:
- put_copyright(C, "//")
-
- put_banner(
- [C],
- ["Automatically generated by tools/codegen/core/gen_experiments.py"],
- "//")
-
- print("#include <grpc/support/port_platform.h>", file=C)
- print("#include \"src/core/lib/experiments/experiments.h\"", file=C)
- print(file=C)
- print("#ifndef GRPC_EXPERIMENTS_ARE_FINAL", file=C)
- print("namespace {", file=C)
- for attr in attrs:
- print("const char* const description_%s = %s;" %
- (attr['name'], c_str(attr['description'])),
- file=C)
- print("const char* const additional_constraints_%s = \"\";" %
- attr['name'],
- file=C)
- have_defaults = set(
- DEFAULTS[rollout_attr['default']] for rollout_attr in rollouts)
- if 'kDefaultForDebugOnly' in have_defaults:
- print("#ifdef NDEBUG", file=C)
- if 'kDefaultForDebugOnly' in have_defaults:
- print("const bool kDefaultForDebugOnly = false;", file=C)
- print("#else", file=C)
- if 'kDefaultForDebugOnly' in have_defaults:
- print("const bool kDefaultForDebugOnly = true;", file=C)
- print("#endif", file=C)
- print("}", file=C)
- print(file=C)
- print("namespace grpc_core {", file=C)
- print(file=C)
- print("const ExperimentMetadata g_experiment_metadata[] = {", file=C)
- for attr in attrs:
- rollout_attr = get_rollout_attr_for_experiment(attr['name'])
- print(
- " {%s, description_%s, additional_constraints_%s, %s, %s}," %
- (c_str(attr['name']), attr['name'], attr['name'],
- DEFAULTS[rollout_attr['default']],
- 'true' if attr.get('allow_in_fuzzing_config', True) else 'false'),
- file=C)
- print("};", file=C)
- print(file=C)
- print("} // namespace grpc_core", file=C)
- print("#endif", file=C)
-
-bzl_to_tags_to_experiments = dict((key, collections.defaultdict(list))
- for key in BZL_LIST_FOR_DEFAULTS.keys()
- if key is not None)
-
-for attr in attrs:
- rollout_attr = get_rollout_attr_for_experiment(attr['name'])
- for tag in attr['test_tags']:
- bzl_to_tags_to_experiments[rollout_attr['default']][tag].append(
- attr['name'])
-
-with open('bazel/experiments.bzl', 'w') as B:
- put_copyright(B, "#")
-
- put_banner(
- [B],
- ["Automatically generated by tools/codegen/core/gen_experiments.py"],
- "#")
-
- print(
- "\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"",
- file=B)
-
- bzl_to_tags_to_experiments = sorted(
- (BZL_LIST_FOR_DEFAULTS[default], tags_to_experiments)
- for default, tags_to_experiments in bzl_to_tags_to_experiments.items()
- if BZL_LIST_FOR_DEFAULTS[default] is not None)
-
- print(file=B)
- print("EXPERIMENTS = {", file=B)
- for key, tags_to_experiments in bzl_to_tags_to_experiments:
- print(" \"%s\": {" % key, file=B)
- for tag, experiments in sorted(tags_to_experiments.items()):
- print(" \"%s\": [" % tag, file=B)
- for experiment in sorted(experiments):
- print(" \"%s\"," % experiment, file=B)
- print(" ],", file=B)
- print(" },", file=B)
- print("}", file=B)
+if not args.disable_gen_bzl:
+ print("Generating experiments.bzl")
+ compiler.GenExperimentsBzl('bazel/experiments.bzl')
