tools/buildgen/generate_projects.sh

#! /bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
echo "System Python interpreter: $(python3 -VV)"

export TEST=${TEST:-false}

# TODO(sergiitk): we should be using a virtual environment for this instead.
YAML_OK=$(python3 -c "import yaml; print(yaml.__version__.split('.') >= ['5', '4', '1'])")

if [[ "${YAML_OK}" != "True" ]]; then
  # PyYAML dropped 3.5 support at 5.4.1, which makes 5.3.1 the latest version we
  # can use.
  python3 -m pip install --upgrade --ignore-installed PyYAML==5.3.1 --user
fi

cd `dirname $0`/../..

echo "Generating build_autogenerated.yaml from bazel BUILD file"
rm -f build_autogenerated.yaml
python3 tools/buildgen/extract_metadata_from_bazel_xml.py

tools/buildgen/build_cleaner.py build_handwritten.yaml

echo "[gRPC Python] Start: auxilary packages"
# Use a virtual environment to work on gRPC Python auxilary packages.
VENV_DIR=".venv-generate-projects"
VENV_PYTHON="${VENV_DIR}/bin/python"
VENV_PIP="${VENV_DIR}/bin/pip"

# Remove existing venv on python version mismatch.
if [[ -d "${VENV_DIR}" ]]; then
  PY_VERSION="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
  PY_VENV_VERSION="$(${VENV_PYTHON} -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' || echo "broken")"
  if  [[ "${PY_VERSION}" !=  "${PY_VENV_VERSION}" ]]; then
    echo "Python version mismatch: system=${PY_VERSION}, venv=${PY_VENV_VERSION}"
    echo "Removing existing venv: ${VENV_DIR}"
    rm -rf "${VENV_DIR}"
  fi
fi

# Create a clean virtual environment.
if [[ ! -d "${VENV_DIR}" ]]; then
  # Attempt to install virtualenv.
  if ! python3 -m pip freeze | grep virtualenv &>/dev/null; then
    echo "virtualenv Python module not installed. Attempting to install via pip." >/dev/stderr
    if INSTALL_OUTPUT=$(! python3 -m pip install virtualenv --upgrade &>/dev/stdout); then
      echo "$INSTALL_OUTPUT"
      if echo "$INSTALL_OUTPUT" | grep "externally managed" &>/dev/null; then
        echo >/dev/stderr
        echo "############################" >/dev/stderr
        echo  "Your administrator is _insisting_ on managing your packages themself. Try running \`sudo apt-get install python3-virtualenv\`" >/dev/stderr
        echo "############################" >/dev/stderr
      fi
      exit 1
    fi
  fi

  echo "Creating a new venv: ${VENV_DIR}"
  python3 -m virtualenv "${VENV_DIR}"
fi

echo -n "Using ${VENV_DIR} venv with Python interpreter: "
"${VENV_PYTHON}" -VV

# TODO(sergiitk): remove GRPC_GENERATE_PROJECTS_SKIP_XDS_PROTOS when all CIs are
#                 are upgraded to use Python 3.9+.
# WARNING! Do not use this variable when running the script directly. This is
# a temporary measure to allow some other scripts scripts that rely on
# generate_projects.sh (f.e. _at_head jobs), to skip this step while they're
# using on older Python versions.
if [[ "${GRPC_GENERATE_PROJECTS_SKIP_XDS_PROTOS:-0}" != "1" ]]; then
  # Keep grpcio-tools version in sync with XDS_PROTOS_GENCODE_GRPC_VERSION
  # in py_xds_protos/setup.py.
  #
  # Explanation: since PR #40518, xds-protos grpc gencode contains a poison pill
  # that enforces grpcio runtime version to be equal or greater
  # to the grpcio-tools version that generated the code.
  #
  # Note: this version needs to be updated periodically to keep up with mainstream
  # python releases, as older grpcio-tools release may not support them.
  "${VENV_PIP}" install grpcio-tools==1.74.0
  "${VENV_PIP}" list

  echo "[gRPC Python] Generating xds-protos"
  "${VENV_PYTHON}" py_xds_protos/build_xds_protos.py
fi

echo "[gRPC Python] Making other grpcio auxilary packages"
"${VENV_PYTHON}" tools/distrib/python/make_grpcio_tools.py
"${VENV_PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py
echo "[gRPC Python] Done: auxilary packages"

# check build_autogenerated.yaml is already in its "clean" form
TEST=true tools/buildgen/build_cleaner.py build_autogenerated.yaml

. tools/buildgen/generate_build_additions.sh
. tools/buildgen/copy_protos.sh

# Instead of generating from a single build.yaml, we've split it into
# - build_handwritten.yaml: manually written metadata
# - build_autogenerated.yaml: generated from bazel BUILD file
python3 tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files "$@"

rm $gen_build_files

tools/artifact_gen/artifact_gen.sh