#!/usr/bin/env python3
"""Creates the Chromium SQLite amalgamation.
The amalgamation is a single large source file (sqlite3.c) containing all
of the SQLite code. More at https://www.sqlite.org/amalgamation.html.
Usage:
generate_amalgamation.py
"""
import argparse
import os
import stat
import subprocess
import sys
import tempfile
from shutil import copyfile, rmtree
from extract_sqlite_api import ProcessSourceFile, header_line, footer_line
# The Chromium SQLite third party directory (i.e. //third_party/sqlite).
_SQLITE_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
# The Chromium SQLite source directory (i.e. //third_party/sqlite/src).
_SQLITE_SRC_DIR = os.path.join(_SQLITE_ROOT_DIR, 'src')
# The .gni file (also used by BUILD.gn when building) which contains all
# flags passed to the `configuration` script and also used for the compile.
_COMMON_CONFIGURATION_FLAGS_GNI_FILE = os.path.join(
_SQLITE_ROOT_DIR, 'sqlite_common_configuration_flags.gni')
_CHROMIUM_CONFIGURATION_FLAGS_GNI_FILE = os.path.join(
_SQLITE_ROOT_DIR, 'sqlite_chromium_configuration_flags.gni')
_DEV_CONFIGURATION_FLAGS_GNI_FILE = os.path.join(
_SQLITE_ROOT_DIR, 'sqlite_dev_configuration_flags.gni')
# The temporary directory where `make configure` and the amalgamation
# is temporarily created.
_TEMP_CONFIG_DIR = tempfile.mkdtemp()
# Set to True to generate a configuration which is compatible for
# running the SQLite tests.
_CONFIGURE_FOR_TESTING = False
def get_amalgamation_dir(config_name):
if config_name == 'chromium':
return os.path.join(_SQLITE_SRC_DIR, 'amalgamation')
elif config_name == 'dev':
return os.path.join(_SQLITE_SRC_DIR, 'amalgamation_dev')
else:
assert False
def _icu_cpp_flags():
"""Return the libicu C++ flags."""
cmd = ['icu-config', '--cppflags']
try:
return subprocess.check_output(cmd)
except Exception:
return ''
def _icu_ld_flags():
"""Return the libicu linker flags."""
cmd = ['icu-config', '--ldflags']
try:
return subprocess.check_output(cmd)
except Exception:
return ''
def _strip_flags_for_testing(flags):
"""Accepts the default configure/build flags and strips out those
incompatible with the SQLite tests.
When configuring SQLite to run tests this script uses a configuration
as close to what Chromium ships as possible. Some flags need to be
omitted for the tests to link and run correct. See comments below.
"""
test_flags = []
for flag in flags:
# Omitting features can cause tests to hang/crash/fail because the
# SQLite tests don't seem to detect feature omission. Keep them enabled.
if flag.startswith('SQLITE_OMIT_'):
continue
# Some tests compile with specific SQLITE_DEFAULT_PAGE_SIZE so do
# not hard-code.
if flag.startswith('SQLITE_DEFAULT_PAGE_SIZE='):
continue
# Some tests compile with specific SQLITE_DEFAULT_MEMSTATUS so do
# not hard-code.
if flag.startswith('SQLITE_DEFAULT_MEMSTATUS='):
continue
# If enabled then get undefined reference to `uregex_open_63' and
# other *_64 functions.
if flag == 'SQLITE_ENABLE_ICU':
continue
# If defined then the fts4umlaut tests fail with the following error:
#
# Error: unknown tokenizer: unicode61
if flag == 'SQLITE_DISABLE_FTS3_UNICODE':
continue
test_flags.append(flag)
return test_flags
def _read_flags(file_name, param_name):
config_globals = dict()
with open(file_name) as input_file:
code = compile(input_file.read(), file_name, 'exec')
exec (code, config_globals)
return config_globals[param_name]
def _read_configuration_values(config_name):
"""Read the configuration flags and return them in an array.
|config_name| is one of "chromium" or "dev".
"""
common_flags = _read_flags(_COMMON_CONFIGURATION_FLAGS_GNI_FILE,
'sqlite_common_configuration_flags')
chromium_flags = _read_flags(_CHROMIUM_CONFIGURATION_FLAGS_GNI_FILE,
'sqlite_chromium_configuration_flags')
dev_flags = _read_flags(_DEV_CONFIGURATION_FLAGS_GNI_FILE,
'sqlite_dev_configuration_flags')
if config_name == 'chromium':
flags = common_flags + chromium_flags
elif config_name == 'dev':
flags = common_flags + dev_flags
else:
print('Incorrect config "%s"' % config_name, file=sys.stderr)
sys.exit(1)
if _CONFIGURE_FOR_TESTING:
flags = _strip_flags_for_testing(flags)
return flags
def _do_configure(config_name):
"""Run the configure script for the SQLite source."""
configure = os.path.join(_SQLITE_SRC_DIR, 'configure')
build_flags = ' '.join(
['-D' + f for f in _read_configuration_values(config_name)])
cflags = '-Os {} {}'.format(build_flags, _icu_cpp_flags())
ldflags = _icu_ld_flags()
cmd = [
configure,
'CFLAGS={}'.format(cflags),
'LDFLAGS={}'.format(ldflags),
'--disable-load-extension',
'--enable-amalgamation',
'--enable-threadsafe',
]
subprocess.check_call(cmd)
if _CONFIGURE_FOR_TESTING:
# Copy the files necessary for building/running tests back
#into the source directory.
files = ['Makefile', 'config.h', 'libtool']
for file_name in files:
copyfile(
os.path.join(_TEMP_CONFIG_DIR, file_name),
os.path.join(_SQLITE_SRC_DIR, file_name))
file_name = os.path.join(_SQLITE_SRC_DIR, 'libtool')
st = os.stat(file_name)
os.chmod(file_name, st.st_mode | stat.S_IEXEC)
def make_aggregate(config_name):
"""Generate the aggregate source files."""
if not os.path.exists(_TEMP_CONFIG_DIR):
os.mkdir(_TEMP_CONFIG_DIR)
try:
os.chdir(_TEMP_CONFIG_DIR)
_do_configure(config_name)
# Chromium compiles 'sqlite3r.c' and 'sqlite3r.h' to use the built-in
# corruption recovery module. These files are then mapped to the standard
# 'sqlite3.c' and 'sqlite3.h' files below. This mapping is required if
# the "SQLITE_HAVE_SQLITE3R" configuration option is specified.
cmd = ['make', 'shell.c', 'sqlite3r.h', 'sqlite3r.c']
subprocess.check_call(cmd)
amalgamation_dir = get_amalgamation_dir(config_name)
if not os.path.exists(amalgamation_dir):
os.mkdir(amalgamation_dir)
readme_dst = os.path.join(amalgamation_dir, 'README.md')
if not os.path.exists(readme_dst):
readme_src = os.path.join(_SQLITE_ROOT_DIR, 'scripts',
'README_amalgamation.md')
copyfile(readme_src, readme_dst)
copyfile(os.path.join(_TEMP_CONFIG_DIR, 'sqlite3r.c'),
os.path.join(amalgamation_dir, 'sqlite3.c'))
copyfile(os.path.join(_TEMP_CONFIG_DIR, 'sqlite3r.h'),
os.path.join(amalgamation_dir, 'sqlite3.h'))
# shell.c must be placed in a different directory from sqlite3.h,
# because it contains an '#include "sqlite3.h"' that we want to resolve
# to our custom //third_party/sqlite/sqlite3.h, not to the sqlite3.h
# produced here.
shell_dir = os.path.join(amalgamation_dir, 'shell')
if not os.path.exists(shell_dir):
os.mkdir(shell_dir)
copyfile(
os.path.join(_TEMP_CONFIG_DIR, 'shell.c'),
os.path.join(shell_dir, 'shell.c'))
finally:
rmtree(_TEMP_CONFIG_DIR)
def extract_sqlite_api(config_name):
amalgamation_dir = get_amalgamation_dir(config_name)
input_file = os.path.join(amalgamation_dir, 'sqlite3.h')
output_file = os.path.join(amalgamation_dir, 'rename_exports.h')
ProcessSourceFile(
api_export_macro='SQLITE_API',
symbol_prefix='chrome_',
header_line=header_line,
footer_line=footer_line,
input_file=input_file,
output_file=output_file)
if __name__ == '__main__':
desc = \
('Create the SQLite amalgamation. The SQLite amalgamation is documented at '
'https://www.sqlite.org/amalgamation.html and is a single large file '
'containing the SQLite source code. Chromium generates the amalgamation with'
' this script to ensure that the configuration parameters are identical to '
'those in the Ninja build file.')
parser = argparse.ArgumentParser(description=desc)
parser.add_argument(
'-t',
'--testing',
action='store_true',
help='Generate an amalgamation for testing (default: false)')
namespace = parser.parse_args()
if namespace.testing:
_CONFIGURE_FOR_TESTING = True
print('Running configure for testing.')
for config_name in ['chromium', 'dev']:
make_aggregate(config_name)
extract_sqlite_api(config_name)
Codebase Browser
chromium