diff --git a/.travis.yml b/.travis.yml index 9a7323443409..64a7a565e254 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,53 +1,44 @@ -branches: - only: - - master - - devel - - "1.4" - - "2.3" - - "2.4" - - "2.5" - - "2.6" - - "2.7" - - "2.8" - - "3.0" - - "3.1" - - "3.2" +#branches: +# only: +# - master +# - devel +# - feature/clang-tidy-format language: cpp -cache: ccache -compiler: g++ -sudo: false -dist: trusty +#cache: ccache -addons: - apt: - sources: - - ubuntu-toolchain-r-test - - george-edison55-trusty-backports - packages: - - g++-5 - - gcc-5 - - binutils-gold - - gdb - - cmake-data - - cmake +#see https://apt.llvm.org/ +matrix: + include: + - name: "clang-format" + compiler: clang++-9 + addons: + apt: + sources: + - [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-xenial-8'] + - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main' + key_url: "https://apt.llvm.org/llvm-snapshot.gpg.key" + packages: [ 'clang-9', 'clang-tools-9', 'clang-tidy-9', 'clang-format-9', 'colordiff'] + env: + - MATRIX_ENV="export CXX=clang++-9 CC=clang-9" +before_install: install: +before_script: + # we need this only to create the compilation databse for clang tidy + # - eval "${MATRIX_ENV}" + # - mkdir build && cd build || exit 1 + # - cmake .. + # - cd .. +script: + - ./utils/clang-format-check -# prepare environment -- export PATH="$HOME/bin:$PATH" -- export CC="$HOME/bin/gcc" -- export CXX="$HOME/bin/g++" -- export CFLAGS="-B$HOME/bin/gold $CFLAGS" -- export CXXFLAGS="-B$HOME/bin/gold $CXXFLAGS" +after_suceess: +after_failure: +after_script: -before_script: "bash -c Installation/travisCI/before_script.sh" -script: "bash -c Installation/travisCI/build.sh" -after_failure: "bash -c Installation/travisCI/after_failure.sh" -after_script: "bash -c Installation/travisCI/after_script.sh" - -notifications: - slack: - secure: JrnDfdroyURrS85HIVsI4xw82taol+lvOJxduxz4T8mQuckaE3ECRYcxX7MzLJfjpeSLST5kttUiZBckHdZ/pnmraZlQ+1/b1VE6k5hFzkbeM0ShjXKTxHdudXaJKuENunMxDAjVWaBaTTh/iy8ZZbKUYQtWLtLfw3xa5zCKVaQ= - on_success: change - on_failure: always +#notifications: +# slack: +# secure: JrnDfdroyURrS85HIVsI4xw82taol+lvOJxduxz4T8mQuckaE3ECRYcxX7MzLJfjpeSLST5kttUiZBckHdZ/pnmraZlQ+1/b1VE6k5hFzkbeM0ShjXKTxHdudXaJKuENunMxDAjVWaBaTTh/iy8ZZbKUYQtWLtLfw3xa5zCKVaQ= +# on_success: change +# on_failure: always diff --git a/README_maintainers.md b/README_maintainers.md index fcee3bcaabf9..22efdcc816b6 100644 --- a/README_maintainers.md +++ b/README_maintainers.md @@ -841,3 +841,21 @@ Currently available analyzers are: ./scripts/examine_results.js -- 'yaml,locateLongRunning' --readFile out/UNITTEST_RESULT.json + +### Tooling + +## clang-format / clang-tidy + +You can get prebuild llvm/clang packages at: +http://releases.llvm.org/download.html + +If you want to build the tools for yourself, because there are no packages for +your platform, then you need to clone +`https://github.com/llvm/llvm-project.git` and configure the build with a +command similar to the following. + +``` +cmake -G Ninja -DCMAKE_INSTALL_PREFIX=$HOME/.local -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" ../llvm-project/llvm/ +ninja -j8 +ninja install +``` diff --git a/compile_commands.json b/compile_commands.json new file mode 120000 index 000000000000..25eb4b2b4b7b --- /dev/null +++ b/compile_commands.json @@ -0,0 +1 @@ +build/compile_commands.json \ No newline at end of file diff --git a/utils/clang-format-changed-only b/utils/clang-format-changed-only new file mode 100755 index 000000000000..8d7ded311002 --- /dev/null +++ b/utils/clang-format-changed-only @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -u + +## locate executable +clang_format="${ARANGODB_CLANG_FORMAT:-""}" +clang_format_version="${ARANGODB_CLANG_FORMAT_VERSION:-"6.0.1"}" + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + for candidate in "$HOME/.local/bin/clang-format" "clang-format-arangodb" "clang-format-6" "clang-format"; do + echo "checking candidate $candidate" + path="$(type -p "${candidate}")" + if [[ -n $path ]]; then + clang_format="$path" + echo "selecting this candidate" + break; + fi + done +else + echo "clang-format provided by environment" +fi + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + echo "using fallback" + clang_format="clang-format" +fi + +## check version +echo "checking version of $clang_format" +version_string=$(${clang_format} --version) +re=".*version ${clang_format_version}.*" +if ! [[ $version_string =~ $re ]]; then + echo "your version: '$version_string' does not match version 6.0.1" + exit 1 +fi + + +diff="$(git diff -U0 --no-color)" +# do final formatting +${clang_format} -i -verbose -style=file ${files_final[@]} +command=( + "${bash_source_dir}/lib/clang-format-diff.py" + --verbose + -i + -binary "${clang_format}" + -style=file + -regex '(lib|tests|arangod)/.*' + -p1 +) + +echo "${command[@]}" +"${command[@]}" <<<"$diff" + +## keep this for now +# ## find relevant files +# files="$( +# find arangod arangosh lib enterprise \ +# -name Zip -prune -o \ +# -type f "(" -name "*.cpp" -o -name "*.h" ")" \ +# "!" "(" -name "tokens.*" -o -name "v8-json.*" -o -name "voc-errors.*" -o -name "grammar.*" -o -name "xxhash.*" -o -name "exitcodes.*" ")" +# )" +# +# git_files=$(git diff --name-only) +# files_final=() +# for gf in ${git_files[@]}; do +# echo "---------------------" +# for f in ${files[@]}; do +# echo "$gf -- $f" +# if [[ $gf == "$f" ]]; then +# files_final+=( "$f" ) +# continue +# fi +# done +# done + diff --git a/utils/clang-format-check b/utils/clang-format-check new file mode 100755 index 000000000000..f403b9247803 --- /dev/null +++ b/utils/clang-format-check @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -u + +# Make any failure in piped commands be reflected in the exit code. +set -o pipefail + +diff_target="${1:-HEAD}" +TRAVIS=${TRAVIS:-false} + +readonly bash_source_dir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +readonly project_dir="$(readlink -f $(dirname $(readlink -f ${BASH_SOURCE[0]}))/..)" + +## locate executable +readonly clang_format_version="${ARANGODB_CLANG_FORMAT_VERSION:-"9.0.1"}" +clang_format="${ARANGODB_CLANG_FORMAT:-""}" + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + for candidate in "$HOME/.local/bin/clang-format" "clang-format-arangodb" "clang-format-9" "clang-format"; do + echo "checking candidate $candidate" + path="$(type -p "${candidate}")" + if [[ -n $path ]]; then + clang_format="$path" + echo "selecting this candidate" + break; + fi + done +else + echo "clang-format provided by environment" +fi + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + echo "using fallback" + clang_format="clang-format" +fi + +## check version +echo "checking version of $clang_format" +version_string=$(${clang_format} --version) +re=".*version ${clang_format_version}.*" +if ! [[ $version_string =~ $re ]]; then + echo "your version: '$version_string' does not match version regular expression '$re'" + exit 1 +fi + +if $TRAVIS; then + diff_target="${TRAVIS_BRANCH}" + if ! $TRAVIS_PULL_REQUEST; then + diff_target="${diff_target}^" + fi + echo "diffing against ${diff_target}" +fi + +diff="$(git diff -U0 --no-color "${diff_target}" )" + + #-regex "'${project_dir}"'/(lib|tests|arangod).*'"'" +command=( + "${bash_source_dir}/lib/clang-format-diff.py" + --verbose + -binary "${clang_format}" + -style=file + -regex '(lib|tests|arangod)/.*' + -p1 +) + +echo "${command[@]}" +if type colordiff 2>/dev/null; then + #need pipefail option + "${command[@]}" <<<"$diff" | colordiff -u3 +else + "${command[@]}" <<<"$diff" +fi diff --git a/utils/clang-format-everything b/utils/clang-format-everything new file mode 100755 index 000000000000..0e2fe4a648c3 --- /dev/null +++ b/utils/clang-format-everything @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -u + +## locate executable +clang_format="${ARANGODB_CLANG_FORMAT:-""}" +clang_format_version="${ARANGODB_CLANG_FORMAT_VERSION:-"6.0.1"}" + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + for candidate in "$HOME/.local/bin/clang-format" "clang-format-arangodb" "clang-format-6" "clang-format"; do + echo "checking candidate $candidate" + path="$(type -p "${candidate}")" + if [[ -n $path ]]; then + clang_format="$path" + echo "selecting this candidate" + break; + fi + done +else + echo "clang-format provided by environment" +fi + +# fallback if nothing is found +if [[ -z $clang_format ]]; then + echo "using fallback" + clang_format="clang-format" +fi + +## check version +echo "checking version of $clang_format" +version_string=$(${clang_format} --version) +re=".*version ${clang_format_version}.*" +if ! [[ $version_string =~ $re ]]; then + echo "your version: '$version_string' does not match version 6.0.1" + exit 1 +fi + + +## find relevant files +files="$( + find arangod arangosh lib enterprise \ + -name Zip -prune -o \ + -type f "(" -name "*.cpp" -o -name "*.h" ")" \ + "!" "(" -name "tokens.*" -o -name "v8-json.*" -o -name "voc-errors.*" -o -name "grammar.*" -o -name "xxhash.*" -o -name "exitcodes.*" ")" +)" + +# do final formatting +${clang_format} -i -verbose -style=file ${files[@]} diff --git a/utils/clang-tidy b/utils/clang-tidy new file mode 100755 index 000000000000..c574b39a6114 --- /dev/null +++ b/utils/clang-tidy @@ -0,0 +1,49 @@ +#!/bin/bash + +#!/usr/bin/env bash +set -u + +## locate executable +clang_tidy="${ARANGODB_CLANG_TIDY:-""}" +clang_tidy_version="${ARANGODB_CLANG_TIDY_VERSION:-"9.0.0"}" + +# fallback if nothing is found +if [[ -z $clang_tidy ]]; then + for candidate in "$HOME/.local/bin/clang-tidy" "clang-tidy-arangodb" "clang-tidy-9" "clang-tidy"; do + echo "checking candidate $candidate" + path="$(type -p "${candidate}")" + if [[ -n $path ]]; then + clang_tidy="$path" + echo "selecting this candidate" + break; + fi + done +else + echo "clang-tidy provided by environment" +fi + +# fallback if nothing is found +if [[ -z $clang_tidy ]]; then + echo "using fallback" + clang_tidy="clang-tidy" +fi + +## check version +echo "checking version of $clang_tidy" +version_string=$(${clang_tidy} --version) +re=".*version ${clang_tidy_version}.*" +if ! [[ $version_string =~ $re ]]; then + echo "your version: '$version_string' does not match version 6.0.1" + exit 1 +fi + +## find relevant files + #-type f "(" -name "*.cpp" -o -name "*.h" ")" \ +files="$( + find arangod arangosh lib enterprise \ + -name Zip -prune -o \ + -type f "(" -name "*.cpp" ")" \ + "!" "(" -name "tokens.*" -o -name "v8-json.*" -o -name "voc-errors.*" -o -name "grammar.*" -o -name "xxhash.*" -o -name "exitcodes.*" ")" +)" + +${clang_tidy} --quiet --checks=-*,clang-analyzer-*,-clang-analyzer-cplusplus* ${files} diff --git a/utils/lib/clang-format-diff.py b/utils/lib/clang-format-diff.py new file mode 100755 index 000000000000..f57e8486bdf7 --- /dev/null +++ b/utils/lib/clang-format-diff.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===------------------------------------------------------------------------===# + +r""" +ClangFormat Diff Reformatter +============================ + +This script reads input from a unified diff and reformats all the changed +lines. This is useful to reformat all the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 --no-color HEAD^ | clang-format-diff.py -p1 -i + svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i + +""" +from __future__ import absolute_import, division, print_function + +import argparse +import difflib +import re +import subprocess +import sys + +if sys.version_info.major >= 3: + from io import StringIO +else: + from io import BytesIO as StringIO + + +def main(): + parser = argparse.ArgumentParser(description= + 'Reformat changed lines in diff. Without -i ' + 'option just output the diff that would be ' + 'introduced.') + parser.add_argument('-i', action='store_true', default=False, + help='apply edits to files instead of displaying a diff') + parser.add_argument('-p', metavar='NUM', default=0, + help='strip the smallest prefix containing P slashes') + parser.add_argument('-regex', metavar='PATTERN', default=None, + help='custom pattern selecting file paths to reformat ' + '(case sensitive, overrides -iregex)') + parser.add_argument('-iregex', metavar='PATTERN', default= + r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto' + r'|protodevel|java)', + help='custom pattern selecting file paths to reformat ' + '(case insensitive, overridden by -regex)') + parser.add_argument('-sort-includes', action='store_true', default=False, + help='let clang-format sort include blocks') + parser.add_argument('-v', '--verbose', action='store_true', + help='be more verbose, ineffective without -i') + parser.add_argument('-style', + help='formatting style to apply (LLVM, Google, Chromium, ' + 'Mozilla, WebKit)') + parser.add_argument('-binary', default='clang-format', + help='location of binary to use for clang-format') + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + status = 0 + for line in sys.stdin: + match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) + if match: + filename = match.group(2) + if filename == None: + continue + + if args.regex is not None: + if not re.match('^%s$' % args.regex, filename): + continue + else: + if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1 + lines_by_file.setdefault(filename, []).extend( + ['-lines', str(start_line) + ':' + str(end_line)]) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.items(): + if args.i and args.verbose: + print('Formatting {}'.format(filename)) + command = [args.binary, filename] + if args.i: + command.append('-i') + if args.sort_includes: + command.append('-sort-includes') + command.extend(lines) + if args.style: + command.extend(['-style', args.style]) + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) + + if not args.i: + with open(filename) as f: + code = f.readlines() + formatted_code = StringIO(stdout).readlines() + diff = difflib.unified_diff(code, formatted_code, + filename, filename, + '(before formatting)', '(after formatting)') + diff_string = ''.join(diff) + if len(diff_string) > 0: + sys.stdout.write(diff_string) + status = 1 + + return status + +if __name__ == '__main__': + sys.exit(main()) diff --git a/utils/reformat.sh b/utils/lib/reformat.sh similarity index 100% rename from utils/reformat.sh rename to utils/lib/reformat.sh