diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94590af..5e33857 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,18 @@ jobs: run: | VERSION=$(echo ${{ github.ref }} | awk -F/ '{print $3}') echo "::set-env name=VERSION::${VERSION}" + - name: release + id: release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.VERSION }} + release_name: "${{ env.VERSION }}" + - name: tools + run: make tools + - name: cli + run: make -C cmd/convox build - name: build run: docker build -t convox/convox:${VERSION} . - name: login @@ -22,10 +34,21 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - name: push run: docker push convox/convox:${VERSION} - - name: release - uses: actions/create-release@v1 + - name: release-cli-linux + uses: actions/upload-release-asset@v1.0.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ env.VERSION }} - release_name: "${{ env.VERSION }}" + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./cmd/convox/pkg/convox-linux-amd64 + asset_name: convox-linux + asset_content_type: application/octet-stream + - name: release-cli-macos + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./cmd/convox/pkg/convox-darwin-10.6-amd64 + asset_name: convox-macos + asset_content_type: application/octet-stream diff --git a/Makefile b/Makefile index 25a0af6..168a6d6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build clean clean-package compress dev generate generate-k8s generate-provider mocks package release test +.PHONY: all build clean clean-package compress dev generate generate-k8s generate-provider mocks package release test tools commands = api atom build router @@ -57,5 +57,8 @@ release: test: env TEST=true go test -covermode atomic -coverprofile coverage.txt -mod=vendor ./... +tools: + go install -mod=vendor ./vendor/github.com/karalabe/xgo + $(binaries): $(GOPATH)/bin/%: $(sources) go install -mod=vendor --ldflags="-s -w" ./cmd/$* diff --git a/cmd/convox/.gitignore b/cmd/convox/.gitignore new file mode 100644 index 0000000..5fff1d9 --- /dev/null +++ b/cmd/convox/.gitignore @@ -0,0 +1 @@ +pkg diff --git a/cmd/convox/Makefile b/cmd/convox/Makefile index 38f4cdc..d20f565 100644 --- a/cmd/convox/Makefile +++ b/cmd/convox/Makefile @@ -1,21 +1,14 @@ -.PHONY: all build clean release release-gopath +.PHONY: all build clean release + +gopath := $(shell mktemp -d) +work := $(gopath)/src/github.com/convox/convox/cmd/convox all: build build: - go install ./... + tar cz -C ../.. . | docker run -v $(gopath)/src/github.com/convox/convox:/convox -i ubuntu tar xz -C /convox + mkdir -p $(work) && cd $(work) && env GOPATH=$(gopath) $(shell go env GOPATH)/bin/xgo -out pkg/convox -targets 'darwin/amd64,linux/amd64' -ldflags "-X main.version=$(VERSION)" . + mkdir -p pkg && docker run -v $(gopath):/gopath -i ubuntu tar czv /gopath/src/github.com/convox/convox/cmd/convox/pkg | tar xzv -C pkg --strip-components 8 clean: - rm -f pkg/convox-* - -release: release-gopath - go get -u github.com/karalabe/xgo - $(GOPATH)/bin/xgo -branch $(shell git rev-parse HEAD) -out pkg/convox -targets 'darwin/amd64,linux/amd64,windows/amd64' -ldflags "-X main.version=$(VERSION)" . - mkdir -p pkg && docker run -v $(GOPATH):/gopath -i ubuntu tar czv /gopath/src/github.com/convox/convox/cmd/convox/pkg | tar xzv -C pkg --strip-components 8 - aws s3 cp pkg/convox-darwin-10.6-amd64 s3://convox/release/$(VERSION)/cli/darwin/convox --acl public-read - aws s3 cp pkg/convox-linux-amd64 s3://convox/release/$(VERSION)/cli/linux/convox --acl public-read - aws s3 cp pkg/convox-windows-4.0-amd64.exe s3://convox/release/$(VERSION)/cli/windows/convox.exe --acl public-read - -# set up gopath in docker volume if running inside a container -release-gopath: - if [ -f /.dockerenv ]; then tar cz $(GOPATH) | docker run -v $(GOPATH):/gopath -i ubuntu tar xz -C /gopath --strip-components 2; fi + rm -f pkg/convox-* \ No newline at end of file diff --git a/docs/guides/installation/cli.md b/docs/guides/installation/cli.md new file mode 100644 index 0000000..08a4e06 --- /dev/null +++ b/docs/guides/installation/cli.md @@ -0,0 +1,13 @@ +# CLI Installation + +## Linux + + $ curl -L https://github.com/convox/convox/releases/latest/download/convox-linux -o /tmp/convox + $ sudo mv /tmp/convox /usr/local/bin/convox + $ sudo chmod 755 /usr/local/bin/convox + +## MacOS + + $ curl -L https://github.com/convox/convox/releases/latest/download/convox-macos -o /tmp/convox + $ sudo mv /tmp/convox /usr/local/bin/convox + $ sudo chmod 755 /usr/local/bin/convox \ No newline at end of file diff --git a/go.mod b/go.mod index e8b729e..76a94be 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/headzoo/ut v0.0.0-20181013193318-a13b5a7a02ca // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf + github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/miekg/dns v1.1.15 github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index a13522e..4dd2791 100644 --- a/go.sum +++ b/go.sum @@ -418,6 +418,8 @@ github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBv github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 h1:AYzjK/SHz6m6mg5iuFwkrAhCc14jvCpW9d6frC9iDPE= +github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7/go.mod h1:iYGcTYIPUvEWhFo6aKUuLchs+AV4ssYdyuBbQJZGcBk= github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/karrick/godirwalk v1.7.7/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/karrick/godirwalk v1.7.8/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= diff --git a/install/README.md b/install/README.md index 1cfb257..099c42a 100644 --- a/install/README.md +++ b/install/README.md @@ -4,9 +4,9 @@ Convox uses [Terraform](https://www.terraform.io/) for installation. Go into the relevant subdirectory of this repository and follow the instructions in the README -| Cloud Provider | Subdirectory | -|:--------------------|:-----------------| -| Amazon Web Services | [aws](aws) | -| Digital Ocean | [do](do) | -| Google Cloud | [gcp](gcp) | -| Microsoft Azure | [azure](azure) | \ No newline at end of file +| Cloud Provider | Subdirectory | +| :------------------ | :------------- | +| Amazon Web Services | [aws](aws) | +| Digital Ocean | [do](do) | +| Google Cloud | [gcp](gcp) | +| Microsoft Azure | [azure](azure) | diff --git a/install/aws/README.md b/install/aws/README.md index 9d5ebfd..d4f9bf8 100644 --- a/install/aws/README.md +++ b/install/aws/README.md @@ -2,8 +2,9 @@ ## Initial Setup -- [Install the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) -- [Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) +- Create an IAM user with the `AdministratorAccess` policy +- Create Access Credentials for this IAM user +- Note these credentials ## Configuration @@ -21,6 +22,6 @@ ## Convox CLI Setup -- [Install the Convox CLI](https://docs.convox.com/introduction/installation) +- [Install the Convox CLI](../../docs/guides/installation/cli.md) - Run `export RACK_URL=$(terraform output rack_url)` - Run `convox rack` to ensure that your CLI is connected to your new Rack \ No newline at end of file diff --git a/install/azure/README.md b/install/azure/README.md index be5bede..904e9d7 100644 --- a/install/azure/README.md +++ b/install/azure/README.md @@ -13,6 +13,6 @@ ## Convox CLI Setup -- [Install the Convox CLI](https://docs.convox.com/introduction/installation) +- [Install the Convox CLI](../../docs/guides/installation/cli.md) - Run `export RACK_URL=$(terraform output rack_url)` - Run `convox rack` to ensure that your CLI is connected to your new Rack diff --git a/install/do/README.md b/install/do/README.md index 720a046..8518877 100644 --- a/install/do/README.md +++ b/install/do/README.md @@ -23,6 +23,6 @@ ## Convox CLI Setup -- [Install the Convox CLI](https://docs.convox.com/introduction/installation) +- [Install the Convox CLI](../../docs/guides/installation/cli.md) - Run `export RACK_URL=$(terraform output rack_url)` - Run `convox rack` to ensure that your CLI is connected to your new Rack diff --git a/install/gcp/README.md b/install/gcp/README.md index 8c5957a..20722d4 100644 --- a/install/gcp/README.md +++ b/install/gcp/README.md @@ -31,6 +31,6 @@ ## Convox CLI Setup -- [Install the Convox CLI](https://docs.convox.com/introduction/installation) +- [Install the Convox CLI](../../docs/guides/installation/cli.md) - Run `export RACK_URL=$(terraform output rack_url)` - Run `convox rack` to ensure that your CLI is connected to your new Rack diff --git a/tools.go b/tools.go index 2e0f6eb..27c7091 100644 --- a/tools.go +++ b/tools.go @@ -3,6 +3,7 @@ package convox import ( + _ "github.com/karalabe/xgo" _ "github.com/vektra/mockery/cmd/mockery" _ "k8s.io/code-generator/cmd/client-gen" _ "k8s.io/code-generator/cmd/conversion-gen" diff --git a/vendor/github.com/karalabe/xgo/LICENSE b/vendor/github.com/karalabe/xgo/LICENSE new file mode 100644 index 0000000..670edf9 --- /dev/null +++ b/vendor/github.com/karalabe/xgo/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2014 Péter Szilágyi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/karalabe/xgo/README.md b/vendor/github.com/karalabe/xgo/README.md new file mode 100644 index 0000000..ed4269c --- /dev/null +++ b/vendor/github.com/karalabe/xgo/README.md @@ -0,0 +1,272 @@ +# xgo - Go CGO cross compiler + +Although Go strives to be a cross platform language, cross compilation from one +platform to another is not as simple as it could be, as you need the Go sources +bootstrapped to each platform and architecture. + +The first step towards cross compiling was Dave Cheney's [golang-crosscompile](https://github.com/davecheney/golang-crosscompile) +package, which automatically bootstrapped the necessary sources based on your +existing Go installation. Although this was enough for a lot of cases, certain +drawbacks became apparent where the official libraries used CGO internally: any +dependency to third party platform code is unavailable, hence those parts don't +cross compile nicely (native DNS resolution, system certificate access, etc). + +A step forward in enabling cross compilation was Alan Shreve's [gonative](https://github.com/inconshreveable/gonative) +package, which instead of bootstrapping the different platforms based on the +existing Go installation, downloaded the official pre-compiled binaries from the +golang website and injected those into the local toolchain. Since the pre-built +binaries already contained the necessary platform specific code, the few missing +dependencies were resolved, and true cross compilation could commence... of pure +Go code. + +However, there was still one feature missing: cross compiling Go code that used +CGO itself, which isn't trivial since you need access to OS specific headers and +libraries. This becomes very annoying when you need access only to some trivial +OS specific functionality (e.g. query the CPU load), but need to configure and +maintain separate build environments to do it. + +## Enter xgo + +My solution to the challenge of cross compiling Go code with embedded C/C++ snippets +(i.e. CGO_ENABLED=1) is based on the concept of [lightweight Linux containers](http://en.wikipedia.org/wiki/LXC). +All the necessary Go tool-chains, C cross compilers and platform headers/libraries +have been assembled into a single Docker container, which can then be called as if +a single command to compile a Go package to various platforms and architectures. + +## Installation + +Although you could build the container manually, it is available as an automatic +trusted build from Docker's container registry (not insignificant in size): + + docker pull karalabe/xgo-latest + +To prevent having to remember a potentially complex Docker command every time, +a lightweight Go wrapper was written on top of it. + + go get github.com/karalabe/xgo + +## Usage + +Simply specify the import path you want to build, and xgo will do the rest: + + $ xgo github.com/project-iris/iris + ... + + $ ls -al + -rwxr-xr-x 1 root root 9995000 Nov 24 16:44 iris-android-16-arm + -rwxr-xr-x 1 root root 6776500 Nov 24 16:44 iris-darwin-10.6-386 + -rwxr-xr-x 1 root root 8755532 Nov 24 16:44 iris-darwin-10.6-amd64 + -rwxr-xr-x 1 root root 7114176 Nov 24 16:45 iris-ios-5.0-arm + -rwxr-xr-x 1 root root 10135248 Nov 24 16:44 iris-linux-386 + -rwxr-xr-x 1 root root 12598472 Nov 24 16:44 iris-linux-amd64 + -rwxr-xr-x 1 root root 10040464 Nov 24 16:44 iris-linux-arm + -rwxr-xr-x 1 root root 7516368 Nov 24 16:44 iris-windows-4.0-386.exe + -rwxr-xr-x 1 root root 9549416 Nov 24 16:44 iris-windows-4.0-amd64.exe + + +If the path is not a canonical import path, but rather a local path (starts with +a dot `.` or a dash `/`), xgo will use the local GOPATH contents for the cross +compilation. + + +### Build flags + +A handful of flags can be passed to `go build`. The currently supported ones are + + - `-v`: prints the names of packages as they are compiled + - `-x`: prints the build commands as compilation progresses + - `-race`: enables data race detection (supported only on amd64, rest built without) + - `-tags='tag list'`: list of build tags to consider satisfied during the build + - `-ldflags='flag list'`: arguments to pass on each go tool link invocation + - `-buildmode=mode`: binary type to produce by the compiler + + +### Go releases + +As newer versions of the language runtime, libraries and tools get released, +these will get incorporated into xgo too as extensions layers to the base cross +compilation image (only Go 1.3 and above will be supported). + +You can select which Go release to work with through the `-go` command line flag +to xgo and if the specific release was already integrated, it will automatically +be retrieved and installed. + + $ xgo -go 1.6.1 github.com/project-iris/iris + +Additionally, a few wildcard release strings are also supported: + + - `latest` will use the latest Go release (this is the default) + - `1.6.x` will use the latest point release of a specific Go version + - `1.6-develop` will use the develop branch of a specific Go version + - `develop` will use the develop branch of the entire Go repository + +### Output prefixing + +xgo by default uses the name of the package being cross compiled as the output +file prefix. This can be overridden with the `-out` flag. + + $ xgo -out iris-v0.3.2 github.com/project-iris/iris + ... + + $ ls -al + -rwxr-xr-x 1 root root 9995000 Nov 24 16:44 iris-v0.3.2-android-16-arm + -rwxr-xr-x 1 root root 6776500 Nov 24 16:44 iris-v0.3.2-darwin-10.6-386 + -rwxr-xr-x 1 root root 8755532 Nov 24 16:44 iris-v0.3.2-darwin-10.6-amd64 + -rwxr-xr-x 1 root root 7114176 Nov 24 16:45 iris-v0.3.2-ios-5.0-arm + -rwxr-xr-x 1 root root 10135248 Nov 24 16:44 iris-v0.3.2-linux-386 + -rwxr-xr-x 1 root root 12598472 Nov 24 16:44 iris-v0.3.2-linux-amd64 + -rwxr-xr-x 1 root root 10040464 Nov 24 16:44 iris-v0.3.2-linux-arm + -rwxr-xr-x 1 root root 7516368 Nov 24 16:44 iris-v0.3.2-windows-4.0-386.exe + -rwxr-xr-x 1 root root 9549416 Nov 24 16:44 iris-v0.3.2-windows-4.0-amd64.exe + + +### Branch selection + +Similarly to `go get`, xgo also uses the `master` branch of a repository during +source code retrieval. To switch to a different branch before compilation pass +the desired branch name through the `--branch` argument. + + $ xgo --branch release-branch.go1.4 golang.org/x/tools/cmd/goimports + ... + + $ ls -al + -rwxr-xr-x 1 root root 4171248 Nov 24 16:40 goimports-android-16-arm + -rwxr-xr-x 1 root root 4139868 Nov 24 16:40 goimports-darwin-10.6-386 + -rwxr-xr-x 1 root root 5186720 Nov 24 16:40 goimports-darwin-10.6-amd64 + -rwxr-xr-x 1 root root 3202364 Nov 24 16:40 goimports-ios-5.0-arm + -rwxr-xr-x 1 root root 4189456 Nov 24 16:40 goimports-linux-386 + -rwxr-xr-x 1 root root 5264136 Nov 24 16:40 goimports-linux-amd64 + -rwxr-xr-x 1 root root 4209416 Nov 24 16:40 goimports-linux-arm + -rwxr-xr-x 1 root root 4348416 Nov 24 16:40 goimports-windows-4.0-386.exe + -rwxr-xr-x 1 root root 5415424 Nov 24 16:40 goimports-windows-4.0-amd64.exe + + +### Remote selection + +Yet again similarly to `go get`, xgo uses the repository remote corresponding to +the import path being built. To switch to a different remote while preserving the +original import path, use the `--remote` argument. + + $ xgo --remote github.com/golang/tools golang.org/x/tools/cmd/goimports + ... + +### Package selection + +If you used the above *branch* or *remote* selection machanisms, it may happen +that the path you are trying to build is only present in the specific branch and +not the default repository, causing Go to fail at locating it. To circumvent this, +you may specify only the repository root for xgo, and use an additional `--pkg` +parameter to select the exact package within, honoring any prior *branch* and +*remote* selections. + + $ xgo --pkg cmd/goimports golang.org/x/tools + ... + + $ ls -al + -rwxr-xr-x 1 root root 4194956 Nov 24 16:38 goimports-android-16-arm + -rwxr-xr-x 1 root root 4164448 Nov 24 16:38 goimports-darwin-10.6-386 + -rwxr-xr-x 1 root root 5223584 Nov 24 16:38 goimports-darwin-10.6-amd64 + -rwxr-xr-x 1 root root 3222848 Nov 24 16:39 goimports-ios-5.0-arm + -rwxr-xr-x 1 root root 4217184 Nov 24 16:38 goimports-linux-386 + -rwxr-xr-x 1 root root 5295768 Nov 24 16:38 goimports-linux-amd64 + -rwxr-xr-x 1 root root 4233120 Nov 24 16:38 goimports-linux-arm + -rwxr-xr-x 1 root root 4373504 Nov 24 16:38 goimports-windows-4.0-386.exe + -rwxr-xr-x 1 root root 5450240 Nov 24 16:38 goimports-windows-4.0-amd64.exe + +This argument may at some point be integrated into the import path itself, but for +now it exists as an independent build parameter. Also, there is not possibility +for now to build mulitple commands in one go. + +### Limit build targets + +By default `xgo` will try and build the specified package to all platforms and +architectures supported by the underlying Go runtime. If you wish to restrict +the build to only a few target systems, use the comma separated `--targets` CLI +argument: + + * `--targets=linux/arm`: builds only the ARMv5 Linux binaries (`arm-6`/`arm-7` allowed) + * `--targets=windows/*,darwin/*`: builds all Windows and OSX binaries + * `--targets=*/arm`: builds ARM binaries for all platforms + * `--targets=*/*`: builds all suppoted targets (default) + +The supported targets are: + + * Platforms: `android`, `darwin`, `ios`, `linux`, `windows` + * Achitectures: `386`, `amd64`, `arm-5`, `arm-6`, `arm-7`, `arm64`, `mips`, `mipsle`, `mips64`, `mips64le` + +### Platform versions + +By default `xgo` tries to cross compile to the lowest possible versions of every +supported platform, in order to produce binaries that are portable among various +versions of the same operating system. This however can lead to issues if a used +dependency is only supported by more recent systems. As such, `xgo` supports the +selection of specific platform versions by appending them to the OS target string. + + * `--targets=ios-8.1/*`: cross compile to iOS 8.1 + * `--targets=android-16/*`: cross compile to Android Jelly Bean + * `--targets=darwin-10.9/*`: cross compile to Mac OS X Mavericks + * `--targets=windows-6.0/*`: cross compile to Windows Vista + +The supported platforms are: + + * All Android APIs up to Android Lollipop 5.0 ([API level ids](https://source.android.com/source/build-numbers.html)) + * All Windows APIs up to Windows 8.1 limited by `mingw-w64` ([API level ids](https://en.wikipedia.org/wiki/Windows_NT#Releases)) + * OSX APIs in the range of 10.6 - 10.11 + * All iOS APIs up to iOS 9.3 + +### Mobile libraries + +Apart from the usual runnable binaries, `xgo` also supports building library +archives for Android (`android/aar`) and iOS (`ios/framework`). Opposed to +`gomobile` however `xgo` does not derive library APIs from the Go code, so +proper CGO C external methods must be defined within the package. + +In the case of Android archives, all architectures will be bundled that are +supported by the requested Android platform version. For iOS frameworks `xgo` +will bundle armv7 and arm64 by default, and also the x86_64 simulator builds +if the iPhoneSimulator.sdk was injected by the user: + +* Create a new docker image based on xgo: `FROM karalabe/xgo-latest` +* Inject the simulator SDK: `ADD iPhoneSimulator9.3.sdk.tar.xz /iPhoneSimulator9.3.sdk.tar.xz` +* Bootstrap the simulator SDK: `$UPDATE_IOS /iPhoneSimulator9.3.sdk.tar.xz` + +### CGO dependencies + +The main differentiator of xgo versus other cross compilers is support for basic +embedded C/C++ code and target-platform specific OS SDK availability. The current +xgo release introduces an experimental CGO *dependency* cross compilation, enabling +building Go programs that require external C/C++ libraries. + +It is assumed that the dependent C/C++ library is `configure/make` based, was +properly prepared for cross compilation and is available as a tarball download +(`.tar`, `.tar.gz` or `.tar.bz2`). Further plans include extending this to cmake +based projects, if need arises (please open an issue if it's important to you). + +Such dependencies can be added via the `--deps` argument. They will be retrieved +prior to starting the cross compilation and the packages cached to save bandwidth +on subsequent calls. + +A complex sample for such a scenario is building the Ethereum CLI node, which has +the GNU Multiple Precision Arithmetic Library as it's dependency. + + $ xgo --deps=https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2 \ + --targets=windows/* github.com/ethereum/go-ethereum/cmd/geth + ... + + $ ls -al + -rwxr-xr-x 1 root root 16315679 Nov 24 16:39 geth-windows-4.0-386.exe + -rwxr-xr-x 1 root root 19452036 Nov 24 16:38 geth-windows-4.0-amd64.exe + +Some trivial arguments may be passed to the dependencies' configure script via +`--depsargs`. + + $ xgo --deps=https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2 \ + --targets=ios/* --depsargs=--disable-assembly \ + github.com/ethereum/go-ethereum/cmd/geth + ... + + $ ls -al + -rwxr-xr-x 1 root root 14804160 Nov 24 16:32 geth-ios-5.0-arm + +Note, that since xgo needs to cross compile the dependencies for each platform +and architecture separately, build time can increase significantly. diff --git a/vendor/github.com/karalabe/xgo/xgo.go b/vendor/github.com/karalabe/xgo/xgo.go new file mode 100644 index 0000000..b2bae9c --- /dev/null +++ b/vendor/github.com/karalabe/xgo/xgo.go @@ -0,0 +1,373 @@ +// Go CGO cross compiler +// Copyright (c) 2014 Péter Szilágyi. All rights reserved. +// +// Released under the MIT license. + +// Wrapper around the GCO cross compiler docker container. +package main + +import ( + "bytes" + "flag" + "fmt" + "go/build" + "io" + "log" + "net/http" + "os" + "os/exec" + "os/user" + "path/filepath" + "strconv" + "strings" +) + +// Path where to cache external dependencies +var depsCache string + +func init() { + // Initialize the external dependency cache path to a few possible locations + if home := os.Getenv("HOME"); home != "" { + depsCache = filepath.Join(home, ".xgo-cache") + return + } + if user, err := user.Current(); user != nil && err == nil && user.HomeDir != "" { + depsCache = filepath.Join(user.HomeDir, ".xgo-cache") + return + } + depsCache = filepath.Join(os.TempDir(), "xgo-cache") +} + +// Cross compilation docker containers +var dockerBase = "karalabe/xgo-base" +var dockerDist = "karalabe/xgo-" + +// Command line arguments to fine tune the compilation +var ( + goVersion = flag.String("go", "latest", "Go release to use for cross compilation") + srcPackage = flag.String("pkg", "", "Sub-package to build if not root import") + srcRemote = flag.String("remote", "", "Version control remote repository to build") + srcBranch = flag.String("branch", "", "Version control branch to build") + outPrefix = flag.String("out", "", "Prefix to use for output naming (empty = package name)") + outFolder = flag.String("dest", "", "Destination folder to put binaries in (empty = current)") + crossDeps = flag.String("deps", "", "CGO dependencies (configure/make based archives)") + crossArgs = flag.String("depsargs", "", "CGO dependency configure arguments") + targets = flag.String("targets", "*/*", "Comma separated targets to build for") + dockerImage = flag.String("image", "", "Use custom docker image instead of official distribution") +) + +// ConfigFlags is a simple set of flags to define the environment and dependencies. +type ConfigFlags struct { + Repository string // Root import path to build + Package string // Sub-package to build if not root import + Prefix string // Prefix to use for output naming + Remote string // Version control remote repository to build + Branch string // Version control branch to build + Dependencies string // CGO dependencies (configure/make based archives) + Arguments string // CGO dependency configure arguments + Targets []string // Targets to build for +} + +// Command line arguments to pass to go build +var ( + buildVerbose = flag.Bool("v", false, "Print the names of packages as they are compiled") + buildSteps = flag.Bool("x", false, "Print the command as executing the builds") + buildRace = flag.Bool("race", false, "Enable data race detection (supported only on amd64)") + buildTags = flag.String("tags", "", "List of build tags to consider satisfied during the build") + buildLdFlags = flag.String("ldflags", "", "Arguments to pass on each go tool link invocation") + buildMode = flag.String("buildmode", "default", "Indicates which kind of object file to build") +) + +// BuildFlags is a simple collection of flags to fine tune a build. +type BuildFlags struct { + Verbose bool // Print the names of packages as they are compiled + Steps bool // Print the command as executing the builds + Race bool // Enable data race detection (supported only on amd64) + Tags string // List of build tags to consider satisfied during the build + LdFlags string // Arguments to pass on each go tool link invocation + Mode string // Indicates which kind of object file to build +} + +func main() { + // Retrieve the CLI flags and the execution environment + flag.Parse() + + xgoInXgo := os.Getenv("XGO_IN_XGO") == "1" + if xgoInXgo { + depsCache = "/deps-cache" + } + // Only use docker images if we're not already inside out own image + image := "" + + if !xgoInXgo { + // Ensure docker is available + if err := checkDocker(); err != nil { + log.Fatalf("Failed to check docker installation: %v.", err) + } + // Validate the command line arguments + if len(flag.Args()) != 1 { + log.Fatalf("Usage: %s [options] ", os.Args[0]) + } + // Select the image to use, either official or custom + image = dockerDist + *goVersion + if *dockerImage != "" { + image = *dockerImage + } + // Check that all required images are available + found, err := checkDockerImage(image) + switch { + case err != nil: + log.Fatalf("Failed to check docker image availability: %v.", err) + case !found: + fmt.Println("not found!") + if err := pullDockerImage(image); err != nil { + log.Fatalf("Failed to pull docker image from the registry: %v.", err) + } + default: + fmt.Println("found.") + } + } + // Cache all external dependencies to prevent always hitting the internet + if *crossDeps != "" { + if err := os.MkdirAll(depsCache, 0751); err != nil { + log.Fatalf("Failed to create dependency cache: %v.", err) + } + // Download all missing dependencies + for _, dep := range strings.Split(*crossDeps, " ") { + if url := strings.TrimSpace(dep); len(url) > 0 { + path := filepath.Join(depsCache, filepath.Base(url)) + + if _, err := os.Stat(path); err != nil { + fmt.Printf("Downloading new dependency: %s...\n", url) + + out, err := os.Create(path) + if err != nil { + log.Fatalf("Failed to create dependency file: %v.", err) + } + res, err := http.Get(url) + if err != nil { + log.Fatalf("Failed to retrieve dependency: %v.", err) + } + defer res.Body.Close() + + if _, err := io.Copy(out, res.Body); err != nil { + log.Fatalf("Failed to download dependency: %v", err) + } + out.Close() + + fmt.Printf("New dependency cached: %s.\n", path) + } else { + fmt.Printf("Dependency already cached: %s.\n", path) + } + } + } + } + // Assemble the cross compilation environment and build options + config := &ConfigFlags{ + Repository: flag.Args()[0], + Package: *srcPackage, + Remote: *srcRemote, + Branch: *srcBranch, + Prefix: *outPrefix, + Dependencies: *crossDeps, + Arguments: *crossArgs, + Targets: strings.Split(*targets, ","), + } + flags := &BuildFlags{ + Verbose: *buildVerbose, + Steps: *buildSteps, + Race: *buildRace, + Tags: *buildTags, + LdFlags: *buildLdFlags, + Mode: *buildMode, + } + folder, err := os.Getwd() + if err != nil { + log.Fatalf("Failed to retrieve the working directory: %v.", err) + } + if *outFolder != "" { + folder, err = filepath.Abs(*outFolder) + if err != nil { + log.Fatalf("Failed to resolve destination path (%s): %v.", *outFolder, err) + } + } + // Execute the cross compilation, either in a container or the current system + if !xgoInXgo { + err = compile(image, config, flags, folder) + } else { + err = compileContained(config, flags, folder) + } + if err != nil { + log.Fatalf("Failed to cross compile package: %v.", err) + } +} + +// Checks whether a docker installation can be found and is functional. +func checkDocker() error { + fmt.Println("Checking docker installation...") + if err := run(exec.Command("docker", "version")); err != nil { + return err + } + fmt.Println() + return nil +} + +// Checks whether a required docker image is available locally. +func checkDockerImage(image string) (bool, error) { + fmt.Printf("Checking for required docker image %s... ", image) + out, err := exec.Command("docker", "images", "--no-trunc").Output() + if err != nil { + return false, err + } + return bytes.Contains(out, []byte(image)), nil +} + +// Pulls an image from the docker registry. +func pullDockerImage(image string) error { + fmt.Printf("Pulling %s from docker registry...\n", image) + return run(exec.Command("docker", "pull", image)) +} + +// compile cross builds a requested package according to the given build specs +// using a specific docker cross compilation image. +func compile(image string, config *ConfigFlags, flags *BuildFlags, folder string) error { + // If a local build was requested, find the import path and mount all GOPATH sources + locals, mounts, paths := []string{}, []string{}, []string{} + if strings.HasPrefix(config.Repository, string(filepath.Separator)) || strings.HasPrefix(config.Repository, ".") { + // Resolve the repository import path from the file path + config.Repository = resolveImportPath(config.Repository) + + // Iterate over all the local libs and export the mount points + if os.Getenv("GOPATH") == "" { + log.Fatalf("No $GOPATH is set or forwarded to xgo") + } + for _, gopath := range strings.Split(os.Getenv("GOPATH"), string(os.PathListSeparator)) { + // Since docker sandboxes volumes, resolve any symlinks manually + sources := filepath.Join(gopath, "src") + filepath.Walk(sources, func(path string, info os.FileInfo, err error) error { + // Skip any folders that errored out + if err != nil { + log.Printf("Failed to access GOPATH element %s: %v", path, err) + return nil + } + // Skip anything that's not a symlink + if info.Mode()&os.ModeSymlink == 0 { + return nil + } + // Resolve the symlink and skip if it's not a folder + target, err := filepath.EvalSymlinks(path) + if err != nil { + return nil + } + if info, err = os.Stat(target); err != nil || !info.IsDir() { + return nil + } + // Skip if the symlink points within GOPATH + if filepath.HasPrefix(target, sources) { + return nil + } + // Folder needs explicit mounting due to docker symlink security + locals = append(locals, target) + mounts = append(mounts, filepath.Join("/ext-go", strconv.Itoa(len(locals)), "src", strings.TrimPrefix(path, sources))) + paths = append(paths, filepath.Join("/ext-go", strconv.Itoa(len(locals)))) + return nil + }) + // Export the main mount point for this GOPATH entry + locals = append(locals, sources) + mounts = append(mounts, filepath.Join("/ext-go", strconv.Itoa(len(locals)), "src")) + paths = append(paths, filepath.Join("/ext-go", strconv.Itoa(len(locals)))) + } + } + // Assemble and run the cross compilation command + fmt.Printf("Cross compiling %s...\n", config.Repository) + + args := []string{ + "run", "--rm", + "-v", folder + ":/build", + "-v", depsCache + ":/deps-cache:ro", + "-e", "REPO_REMOTE=" + config.Remote, + "-e", "REPO_BRANCH=" + config.Branch, + "-e", "PACK=" + config.Package, + "-e", "DEPS=" + config.Dependencies, + "-e", "ARGS=" + config.Arguments, + "-e", "OUT=" + config.Prefix, + "-e", fmt.Sprintf("FLAG_V=%v", flags.Verbose), + "-e", fmt.Sprintf("FLAG_X=%v", flags.Steps), + "-e", fmt.Sprintf("FLAG_RACE=%v", flags.Race), + "-e", fmt.Sprintf("FLAG_TAGS=%s", flags.Tags), + "-e", fmt.Sprintf("FLAG_LDFLAGS=%s", flags.LdFlags), + "-e", fmt.Sprintf("FLAG_BUILDMODE=%s", flags.Mode), + "-e", "TARGETS=" + strings.Replace(strings.Join(config.Targets, " "), "*", ".", -1), + } + for i := 0; i < len(locals); i++ { + args = append(args, []string{"-v", fmt.Sprintf("%s:%s:ro", locals[i], mounts[i])}...) + } + args = append(args, []string{"-e", "EXT_GOPATH=" + strings.Join(paths, ":")}...) + + args = append(args, []string{image, config.Repository}...) + return run(exec.Command("docker", args...)) +} + +// compileContained cross builds a requested package according to the given build +// specs using the current system opposed to running in a container. This is meant +// to be used for cross compilation already from within an xgo image, allowing the +// inheritance and bundling of the root xgo images. +func compileContained(config *ConfigFlags, flags *BuildFlags, folder string) error { + // If a local build was requested, resolve the import path + local := strings.HasPrefix(config.Repository, string(filepath.Separator)) || strings.HasPrefix(config.Repository, ".") + if local { + config.Repository = resolveImportPath(config.Repository) + } + // Fine tune the original environment variables with those required by the build script + env := []string{ + "REPO_REMOTE=" + config.Remote, + "REPO_BRANCH=" + config.Branch, + "PACK=" + config.Package, + "DEPS=" + config.Dependencies, + "ARGS=" + config.Arguments, + "OUT=" + config.Prefix, + fmt.Sprintf("FLAG_V=%v", flags.Verbose), + fmt.Sprintf("FLAG_X=%v", flags.Steps), + fmt.Sprintf("FLAG_RACE=%v", flags.Race), + fmt.Sprintf("FLAG_TAGS=%s", flags.Tags), + fmt.Sprintf("FLAG_LDFLAGS=%s", flags.LdFlags), + fmt.Sprintf("FLAG_BUILDMODE=%s", flags.Mode), + "TARGETS=" + strings.Replace(strings.Join(config.Targets, " "), "*", ".", -1), + } + if local { + env = append(env, "EXT_GOPATH=/non-existent-path-to-signal-local-build") + } + // Assemble and run the local cross compilation command + fmt.Printf("Cross compiling %s...\n", config.Repository) + + cmd := exec.Command("/build.sh", config.Repository) + cmd.Env = append(os.Environ(), env...) + + return run(cmd) +} + +// resolveImportPath converts a package given by a relative path to a Go import +// path using the local GOPATH environment. +func resolveImportPath(path string) string { + abs, err := filepath.Abs(path) + if err != nil { + log.Fatalf("Failed to locate requested package: %v.", err) + } + stat, err := os.Stat(abs) + if err != nil || !stat.IsDir() { + log.Fatalf("Requested path invalid.") + } + pack, err := build.ImportDir(abs, build.FindOnly) + if err != nil { + log.Fatalf("Failed to resolve import path: %v.", err) + } + return pack.ImportPath +} + +// Executes a command synchronously, redirecting its output to stdout. +func run(cmd *exec.Cmd) error { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 229e70d..bb8c382 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -284,6 +284,8 @@ github.com/jmespath/go-jmespath github.com/joho/godotenv # github.com/json-iterator/go v1.1.7 github.com/json-iterator/go +# github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 +github.com/karalabe/xgo # github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kballard/go-shellquote # github.com/konsorten/go-windows-terminal-sequences v1.0.2