Add source import tool

pull/810/head
klzgrad 2019-01-14 00:42:27 -05:00
parent 9d9c1d7e9e
commit c6daed5044
12 changed files with 472 additions and 0 deletions

1
CHROMIUM_VERSION 100644
View File

@ -0,0 +1 @@
149.0.7827.114

101
tools/build_test.sh 100755
View File

@ -0,0 +1,101 @@
#!/bin/sh
set -ex
cd src
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
ccache -C
./get-clang.sh
for i in x64 x86 arm64 arm mipsel mips64el; do
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
export EXTRA_FLAGS="target_cpu=\"$i\""
./get-clang.sh
done
for i in x64 x86 arm64 arm; do
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\""
./get-clang.sh
done
config_openwrt() {
arch="$1"
openwrt="$2"
target_cpu="$3"
extra="$4"
export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra"
export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt"
./get-clang.sh
}
config_openwrt x86_64 'target=x86 subtarget=64' x64
config_openwrt x86 'target=x86 subtarget=generic' x86
config_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"'
config_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"'
config_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64
config_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false'
config_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
config_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
config_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false'
config_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false'
config_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
config_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false'
config_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
config_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false'
config_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false'
config_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false'
rm -f /tmp/trace
inotifywait -m -r -o/tmp/trace --format '%w%f %e' . &
pid=$!
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
./build.sh
for i in x64 x86 arm64 arm mipsel mips64el; do
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
export EXTRA_FLAGS="target_cpu=\"$i\""
./build.sh
done
for i in x64 x86 arm64 arm; do
unset EXTRA_FLAGS
unset OPENWRT_FLAGS
export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\""
./build.sh
done
build_openwrt() {
arch="$1"
openwrt="$2"
target_cpu="$3"
extra="$4"
export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra"
export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt"
./build.sh
}
build_openwrt x86_64 'target=x86 subtarget=64' x64
build_openwrt x86 'target=x86 subtarget=generic' x86
build_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"'
build_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"'
build_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64
build_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false'
build_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
build_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
build_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false'
build_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false'
build_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
build_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false'
build_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
build_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false'
build_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false'
build_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false'
kill $pid

View File

@ -0,0 +1,5 @@
#!/bin/sh
for i in /tmp/trace.*; do
cut -d' ' -f1 $i | LC_ALL=C sort -u | sed 's/\/$//' | LC_ALL=C sort -u >$i.sorted
done
cat /tmp/trace.*.sorted | LC_ALL=C sort -u >/tmp/detected-files

32
tools/exclude.txt 100644
View File

@ -0,0 +1,32 @@
.gitignore
*_unittest.cc
*_unittest.mm
*_unittest.nc
*_perftest.cc
*_fuzztest.cc
*[!s]_test.cc
*fuzzer.[!g]*
*fuzz/*
*fuzzing/*
*[a-hj-z]s_test.cc
*org/chromium*
*.golden
*.javap*
*.pyc
build/linux/debian_*
base/tracing/test
net/data/[!s]*
net/data/s[!s]*
net/data/ssl/[!ce]*
net/data/ssl/c[!h]*
net/http/transport_security_state_static.json
net/third_party/nist-pkits
third_party/boringssl/src/crypto/*.txt
third_party/boringssl/src/crypto/hpke/test-vectors.json
third_party/boringssl/src/crypto/cipher_extra/test
third_party/boringssl/src/pki/testdata
third_party/boringssl/src/ssl/test
third_party/boringssl/src/util
third_party/boringssl/src/third_party/googletest
third_party/boringssl/src/third_party/wycheproof_testvectors
third_party/libc++/src/test

View File

@ -0,0 +1,28 @@
#!/bin/sh
set -ex
have_version=$(cut -d= -f2 src/chrome/VERSION | tr '\n' . | cut -d. -f1-4)
want_version=$(cat CHROMIUM_VERSION)
if [ "$have_version" = "$want_version" ]; then
exit 0
fi
name="chromium-$want_version"
tarball="$name.tar.xz"
url="https://commondatastorage.googleapis.com/chromium-browser-official/$tarball"
root=$(git rev-list --max-parents=0 HEAD)
branch=$(git branch --show-current)
git config core.autocrlf false
git config core.safecrlf false
git -c advice.detachedHead=false checkout $root
rm -rf src
git checkout "$branch" -- tools
sed -i "s/^\^/$name\//" tools/include.txt
if [ -f "/tmp/$tarball" ]; then
cat "/tmp/$tarball" | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt
else
curl "$url" -o- | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt
fi
mv "$name" src
git rm --quiet --force -r tools
git add src
git commit --quiet --amend -m "Import $name" --date=now
git rebase --onto HEAD "$root" "$branch"

90
tools/include.txt 100644
View File

@ -0,0 +1,90 @@
^.clang-format
^.gitattributes
^.gitignore
^.gn
^AUTHORS
^BUILD.gn
^DEPS
^LICENSE
^base
^build
^build_overrides/build.gni
^build_overrides/partition_alloc.gni
^build_overrides/protobuf.gni
^buildtools/deps_revisions.gni
^buildtools/third_party/libc++
^buildtools/third_party/libc++abi/BUILD.gn
^buildtools/third_party/libc++abi/cxa_demangle_stub.cc
^buildtools/third_party/libunwind/BUILD.gn
^chrome/VERSION
^chrome/android/profiles/newest.txt
^chrome/app/app-Info.plist
^chrome/app/theme/chromium/BRANDING
^chrome/build/*.txt
^chrome/version.gni
^components/cbor
^components/network_time
^components/unexportable_keys
^components/version_info
^components/embedder_support/BUILD.gn
^components/embedder_support/*.cc
^components/embedder_support/*.h
^crypto
^extensions/buildflags
^ios/features.gni
^net
^testing/gtest/include/gtest/gtest_prod.h
^third_party/abseil-cpp
^third_party/angle/dotfile_settings.gni
^third_party/angle/src/commit_id.py
^third_party/angle/scripts/file_exists.py
^third_party/apple_apsl
^third_party/blink/public/common/user_agent
^third_party/blink/public/common/common_export.h
^third_party/boringssl
^third_party/brotli
^third_party/closure_compiler/closure_args.gni
^third_party/closure_compiler/compile_js.gni
^third_party/compiler-rt/BUILD.gn
^third_party/compiler-rt/src/lib/builtins/assembly.h
^third_party/compiler-rt/src/lib/builtins/atomic.c
^third_party/compiler-rt/src/lib/builtins/int_endianness.h
^third_party/cpu_features
^third_party/depot_tools/cpplint.py
^third_party/depot_tools/download_from_google_storage.py
^third_party/depot_tools/gn_helper.py
^third_party/depot_tools/subprocess2.py
^third_party/googletest/BUILD.gn
^third_party/googletest/src/googletest/include/gtest/gtest_prod.h
^third_party/icu/config.gni
^third_party/jni_zero
^third_party/libc++
^third_party/libc++abi
^third_party/libunwind
^third_party/llvm-libc
^third_party/lss/linux_syscall_support.h
^third_party/modp_b64
^third_party/nasm
^third_party/perfetto/BUILD.gn
^third_party/perfetto/gn
^third_party/perfetto/include
^third_party/perfetto/protos
^third_party/perfetto/src
^third_party/protobuf/BUILD.gn
^third_party/protobuf/proto_library.gni
^third_party/protobuf/proto_sources.gni
^third_party/protobuf/src
^third_party/protobuf/third_party/utf8_range
^third_party/simdutf
^third_party/zlib
^third_party/zstd
^tools/cfi
^tools/clang/scripts/update.py
^tools/grit
^tools/gritsettings
^tools/metrics
^tools/protoc_wrapper
^tools/ubsan
^tools/update_pgo_profiles.py
^tools/win/DebugVisualizers
^url

View File

@ -0,0 +1,60 @@
#!/bin/sh
# $version can be 21.02 or 19.07.
version=19.07.7
if [ ! -d /tmp/openwrt ]; then
cd /tmp
git clone https://github.com/openwrt/openwrt.git
cd openwrt
fi
cd /tmp/openwrt
git -c advice.detachedHead=false checkout v$version
export TOPDIR=$PWD
cd target/linux
>targets.git
for target in *; do
[ -d $target ] || continue
subtargets=$(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS 2>/dev/null)
[ "$subtargets" ] || subtargets=generic
for subtarget in $subtargets; do
echo $(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET=$subtarget 2>/dev/null | egrep '^(Target:|Target-Arch-Packages:)' | cut -d: -f2) >>targets.git
done
done
targets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/ | grep '<td class="n"><a href=' | cut -d'"' -f4 | sed 's,/,,')
>targets.sdk
for target in $targets; do
subtargets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/$target/ | grep '<td class="n"><a href=' | cut -d'"' -f4 | sed 's,/,,')
for subtarget in $subtargets; do
arch=$(curl -s https://downloads.openwrt.org/releases/$version/targets/$target/$subtarget/profiles.json | grep arch_packages | cut -d'"' -f4)
echo $target/$subtarget $arch >>targets.sdk
done
done
cat >parse-targets.py <<EOF
arch_by_target_git = {}
arch_by_target_sdk = {}
for line in open('targets.git'):
fields = line.split()
if not fields:
continue
arch_by_target_git[fields[0]] = fields[1]
for line in open('targets.sdk'):
fields = line.split()
if len(fields) == 2:
if arch_by_target_git[fields[0]] != fields[1]:
raise Exception(line + ': wrong arch')
arch_by_target_sdk[fields[0]] = fields[1]
else:
arch_by_target_sdk[fields[0]] = ''
for arch in sorted(set(arch_by_target_git.values())):
targets = []
for t in arch_by_target_git:
if arch_by_target_git[t] != arch:
continue
if t in arch_by_target_sdk:
targets.append(t)
else:
targets.append('~~' + t + '~~')
print('|', arch, '|?|', ' '.join(sorted(set(targets))), '|')
EOF
python3 parse-targets.py

View File

@ -0,0 +1,10 @@
#!/bin/sh
if [ ! "$1" ]; then
echo "Usage: $0 PCAP_FILE"
exit 1
fi
file="$1"
# Remember to disable segmentation offload so pcap files won't capture packets larger than MTU:
# sudo ethtool --offload eth0 gso off gro off
tshark -2 -r "$file" -R tls.handshake.extensions_server_name -T fields -e tls.handshake.extensions_server_name -e tcp.stream

View File

@ -0,0 +1,62 @@
#!/usr/bin/env python3
import os
import sys
import subprocess
import json
import xml.etree.ElementTree as ET
if len(sys.argv) != 3:
print(f'Usage: {sys.argv[0]} PCAP_FILE DOMAIN')
os.exit(1)
file = sys.argv[1]
domain = sys.argv[2]
result = subprocess.run(['tshark', '-2', '-r', file, '-q', '-o','tls.keylog_file:/tmp/keys','-Y',
f'http2.header.value == "{domain}"', '-T', 'json'], capture_output=True, check=True, text=True)
json_result = json.loads(result.stdout)
target_tcp_stream = json_result[0]["_source"]["layers"]["tcp"]["tcp.stream"]
result = subprocess.run(['tshark', '-2', '-r', file, '-q', '-o','tls.keylog_file:/tmp/keys','-Y',
f'tcp.stream == {target_tcp_stream}', '-T', 'pdml'], capture_output=True, check=True, text=True)
pdml_result = ET.fromstring(result.stdout)
def children(e, cname):
return [c for c in e if c.attrib['name'] == cname]
start_time = None
for packet in pdml_result:
frame = children(packet, "frame")[0]
frame_number = children(frame, "frame.number")[0].attrib['show']
frame_time_relative = children(frame, "frame.time_relative")[0].attrib['show']
tcp = children(packet, "tcp")[0]
tcp_srcport = children(tcp, "tcp.srcport")[0].attrib['show']
tcp_dstport = children(tcp, "tcp.dstport")[0].attrib['show']
if tcp_dstport == "443":
dir = ''
else:
dir = ''
if start_time is None:
start_time = float(frame_time_relative)
frame_time_relative = float(frame_time_relative) - start_time
http2s = children(packet, "http2")
if len(http2s) == 0:
continue
http2s_desc = []
for http2 in http2s:
http2_stream = children(http2, "http2.stream")
assert len(http2_stream) == 1
http2_stream = http2_stream[0]
http2_magic = children(http2_stream, "http2.magic")
if http2_magic:
http2s_desc.append('Magic')
continue
http2_type = children(http2_stream, "http2.type")[0].attrib['showname'].split(' ')[1]
http2_length = children(http2_stream, "http2.length")[0].attrib['show']
http2_streamid = children(http2_stream, "http2.streamid")[0].attrib['show']
http2_stream_desc = f'{http2_length}:{http2_type}[{http2_streamid}]'
if http2_type == 'HEADERS':
http2_stream_desc += ": " + http2_stream.attrib['showname'].split(',')[-1].strip()
http2s_desc.append(http2_stream_desc)
if not http2s_desc:
continue
print(frame_number, f'{frame_time_relative:.4f}', dir, ', '.join(http2s_desc))

View File

@ -0,0 +1,28 @@
#!/bin/sh
if [ ! "$1" ]; then
echo "Usage: $0 DOMAIN"
exit 1
fi
domain="$1"
sudo echo
tempdir=$(mktemp -d)
rm -f /tmp/keys
sudo rm -f /tmp/direct.pcapng
sudo tshark -Q -a duration:10 -w /tmp/direct.pcapng &
tsharkpid=$!
sleep 1
google-chrome --user-data-dir="$tempdir" --ssl-key-log-file=/tmp/keys --no-default-browser-check --no-first-run --disable-quic "https://$domain/" &
chromepid=$!
sleep 10
kill $chromepid
rm -rf "$tempir"
wait $tsharkpid
sudo chmod +r /tmp/direct.pcapng
./parse-pcap-stream.py /tmp/direct.pcapng "$domain"

View File

@ -0,0 +1,33 @@
#!/bin/sh
if [ ! "$1" ]; then
echo "Usage: $0 USERPASS DOMAIN"
exit 1
fi
userpass="$1"
domain="$2"
sudo echo
rm -f /tmp/keys
sudo tshark -Q -a duration:10 -w /tmp/direct.pcapng &
tsharkpid=$!
sleep 1
../src/out/Release/naive --listen=socks://127.0.0.1:1081 --proxy=https://$userpass@$domain --ssl-key-log-file=/tmp/keys --log &
naivepid=$!
sleep 1
curl -s --proxy socks5h://127.0.0.1:1081 https://www.google.com/ -o/dev/null
sleep 3
kill -INT $naivepid
kill -TERM $naivepid
sleep 1
kill -9 $naivepid
wait $tsharkpid
sudo chmod +r /tmp/direct.pcapng
./parse-pcap-stream.py /tmp/direct.pcapng "$domain"

View File

@ -0,0 +1,22 @@
#!/bin/sh
if [ ! "$1" ]; then
echo "Usage: $0 IFACE"
exit 1
fi
iface="$1"
sudo echo
sudo tcpdump -i "$1" -s0 -w microsoft-direct.pcap &
sleep 1
curl https://www.example.com/
sleep 1
sudo pkill tcpdump
sudo tcpdump -i "$1" -s0 -w microsoft-proxy.pcap &
sleep 1
curl --proxy socks5h://127.0.0.1:1080 https://www.example.com/
sleep 1
sudo pkill tcpdump &