Unverified Commit c736888e authored by Darren Shepherd's avatar Darren Shepherd Committed by GitHub

Merge pull request #1034 from ibuildthecloud/dqlite2

dqlite support
parents 16ec08d4 faab6b4c
FROM golang:1.12.12-alpine3.10
RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make
python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
RUN pip3 install 'tox==3.6.0'
RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static
RUN mkdir -p /go/src/golang.org/x && \
......@@ -17,6 +17,13 @@ RUN if [ "${ARCH}" == "amd64" ]; then \
curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \
fi
ARG DQLITE=true
ENV DQLITE $DQLITE
RUN if [ "$DQLITE" = true ]; then \
curl -sfL https://github.com/rancher/dqlite-build/releases/download/v1.1.0-r5/dqlite-${ARCH}.tgz | tar xzf - -C / && \
apk add --allow-untrusted /usr/local/packages/*.apk \
;fi
ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE
ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
......
......@@ -4,6 +4,7 @@ go 1.12
replace (
github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.7-0.20190926181021-82c7525d98c8
github.com/canonical/go-dqlite => github.com/rancher/go-dqlite v1.1.0-k3s.1
github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20181101203652-af5082808c83
github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601
github.com/containerd/console => github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
......@@ -93,7 +94,7 @@ require (
github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
github.com/lib/pq v1.1.1
github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455
github.com/mattn/go-sqlite3 v1.10.0
github.com/mattn/go-sqlite3 v1.11.0
github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/opencontainers/runc v1.0.0-rc2.0.20190611121236-6cc515888830
......
......@@ -80,8 +80,6 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 h1:D21IyuvjDCshj1/qq+pCNd3VZOAEI9jy6Bi131YlXgI=
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E=
github.com/canonical/go-dqlite v1.1.0 h1:vXGVhHrql++q038JVZk17/VZDbvbH5ySWcObWjuxiBQ=
github.com/canonical/go-dqlite v1.1.0/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y=
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY=
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
......@@ -476,6 +474,8 @@ github.com/mattn/go-shellwords v1.0.5 h1:JhhFTIOslh5ZsPrpa3Wdg8bF0WI3b44EMblmU9w
github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
......@@ -588,6 +588,8 @@ github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07 h1:wR1hn
github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s=
github.com/rancher/flannel v0.11.0-k3s.1 h1:mIwnfWDafjzQgFkZeJ1AkFrrAT3EdBaA1giE0eLJKo8=
github.com/rancher/flannel v0.11.0-k3s.1/go.mod h1:Hn4ZV+eq0LhLZP63xZnxdGwXEoRSxs5sxELxu27M3UA=
github.com/rancher/go-dqlite v1.1.0-k3s.1 h1:w3ghNkY5vqRnnrcqxvHkpBQr6E+R/nIwJfaGdNgJAiw=
github.com/rancher/go-dqlite v1.1.0-k3s.1/go.mod h1:lj8UhpkZddn/Ag0tBsnkbELbxHGMpzrZLMs/nW9/DX4=
github.com/rancher/helm-controller v0.2.2 h1:MUqisy53/Ay1EYOF2uTCYBbGpgtZLNKKrI01BdxIbQo=
github.com/rancher/helm-controller v0.2.2/go.mod h1:0JkL0UjxddNbT4FmLoESarD4Mz8xzA5YlejqJ/U4g+8=
github.com/rancher/kine v0.2.1 h1:pK7QJUVA+/oU6esxKa/LIlBbeLl2HGWIwmu8xrROukQ=
......
......@@ -25,6 +25,7 @@ func NewApp() *cli.App {
Name: "debug",
Usage: "Turn on debug logs",
Destination: &debug,
EnvVar: "K3S_DEBUG",
},
}
......
......@@ -24,10 +24,16 @@ LDFLAGS="
STATIC="
-extldflags '-static'
"
if [ "$DQLITE" = "true" ]; then
DQLITE_TAGS="dqlite"
DQLITE_STATIC_SQLITE="-luv -lraft -lco"
fi
STATIC_SQLITE="
-extldflags '-static -lm -ldl -lz -lpthread'
-extldflags '-static -lm -ldl -lz -lpthread $DQLITE_STATIC_SQLITE'
"
TAGS="ctrd apparmor seccomp no_btrfs netgo osusergo providerless"
TAGS="ctrd apparmor seccomp no_btrfs netgo osusergo providerless $DQLITE_TAGS"
if [ "$STATIC_BUILD" != "true" ]; then
STATIC="
......
......@@ -43,4 +43,7 @@ LDFLAGS="
-w -s
"
STATIC="-extldflags '-static'"
CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
if [ "$DQLITE" = "true" ]; then
DQLITE_TAGS="dqlite"
fi
CGO_ENABLED=0 go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
......@@ -49,6 +49,7 @@ mkdir -p ${OUTPUT}
docker run -d --name ${K3S_SERVER} --privileged \
-p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
-e K3S_CLUSTER_SECRET=${SECRET} \
-e K3S_DEBUG=true \
${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
......
......@@ -89,14 +89,6 @@ type Node C.dqlite_node
func Init() error {
// FIXME: ignore SIGPIPE, see https://github.com/joyent/libuv/issues/1254
C.signal(C.SIGPIPE, C.SIG_IGN)
// Don't enable single thread mode when running tests. TODO: find a
// better way to expose this functionality.
if os.Getenv("GO_DQLITE_MULTITHREAD") == "1" {
return nil
}
if rc := C.initializeSQLite(); rc != 0 {
return fmt.Errorf("%d", rc)
}
return nil
}
......
......@@ -8,34 +8,24 @@ addons:
apt:
update: true
env:
matrix:
- GOTAGS=
- GOTAGS=libsqlite3
- GOTAGS="sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify"
- GOTAGS=sqlite_vacuum_full
go:
- 1.9.x
- 1.10.x
- 1.11.x
- master
before_install:
- |
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew update
fi
- |
go get github.com/smartystreets/goconvey
if [[ "${GOOS}" != "windows" ]]; then
go get github.com/mattn/goveralls
go get golang.org/x/tools/cmd/cover
fi
- go get github.com/smartystreets/goconvey
- go get github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover
script:
- GOOS=$(go env GOOS) GOARCH=$(go env GOARCH) go build -v -tags "${GOTAGS}" .
- |
if [[ "${GOOS}" != "windows" ]]; then
$HOME/gopath/bin/goveralls -repotoken 3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx
go test -race -v . -tags "${GOTAGS}"
fi
- $HOME/gopath/bin/goveralls -repotoken 3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx
- go test -race -v . -tags ""
- go test -race -v . -tags "libsqlite3"
- go test -race -v . -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify"
- go test -race -v . -tags "sqlite_vacuum_full"
\ No newline at end of file
......@@ -10,9 +10,7 @@ go-sqlite3
sqlite3 driver conforming to the built-in database/sql interface
Supported Golang version:
- 1.9.x
- 1.10.x
Supported Golang version: See .travis.yml
[This package follows the official Golang Release Policy.](https://golang.org/doc/devel/release.html#policy)
......@@ -249,7 +247,7 @@ Required dependency
brew install sqlite3
```
For OSX there is an additional package install which is required if you whish to build the `icu` extension.
For OSX there is an additional package install which is required if you wish to build the `icu` extension.
This additional package can be installed with `homebrew`.
......@@ -282,7 +280,7 @@ To compile this package on Windows OS you must have the `gcc` compiler installed
3) Open a terminal for the TDM-GCC toolchain, can be found in the Windows Start menu.
4) Navigate to your project folder and run the `go build ...` command for this package.
For example the TDM-GCC Toolchain can be found [here](ttps://sourceforge.net/projects/tdm-gcc/).
For example the TDM-GCC Toolchain can be found [here](https://sourceforge.net/projects/tdm-gcc/).
## Errors
......@@ -458,15 +456,19 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial
Why is it racy if I use a `sql.Open("sqlite3", ":memory:")` database?
Each connection to :memory: opens a brand new in-memory sql database, so if
Each connection to `":memory:"` opens a brand new in-memory sql database, so if
the stdlib's sql engine happens to open another connection and you've only
specified ":memory:", that connection will see a brand new database. A
workaround is to use "file::memory:?mode=memory&cache=shared". Every
connection to this string will point to the same in-memory database.
specified `":memory:"`, that connection will see a brand new database. A
workaround is to use `"file::memory:?cache=shared"` (or `"file:foobar?mode=memory&cache=shared"`). Every
connection to this string will point to the same in-memory database.
Note that if the last database connection in the pool closes, the in-memory database is deleted. Make sure the [max idle connection limit](https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns) is > 0, and the [connection lifetime](https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) is infinite.
For more information see
* [#204](https://github.com/mattn/go-sqlite3/issues/204)
* [#511](https://github.com/mattn/go-sqlite3/issues/511)
* https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases
* https://www.sqlite.org/inmemorydb.html#sharedmemdb
- Reading from database with large amount of goroutines fails on OSX.
......@@ -481,11 +483,11 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial
You need to implement the feature or call the sqlite3 cli.
More infomation see [#305](https://github.com/mattn/go-sqlite3/issues/305)
More information see [#305](https://github.com/mattn/go-sqlite3/issues/305)
- Error: `database is locked`
When you get an database is locked. Please use the following options.
When you get a database is locked. Please use the following options.
Add to DSN: `cache=shared`
......@@ -497,7 +499,7 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial
Second please set the database connections of the SQL package to 1.
```go
db.SetMaxOpenConn(1)
db.SetMaxOpenConns(1)
```
More information see [#209](https://github.com/mattn/go-sqlite3/issues/209)
......
......@@ -368,7 +368,7 @@ func callbackRet(typ reflect.Type) (callbackRetConverter, error) {
func callbackError(ctx *C.sqlite3_context, err error) {
cstr := C.CString(err.Error())
defer C.free(unsafe.Pointer(cstr))
C.sqlite3_result_error(ctx, cstr, -1)
C.sqlite3_result_error(ctx, cstr, C.int(-1))
}
// Test support code. Tests are not allowed to import "C", so we can't
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -683,7 +683,7 @@ func (c *SQLiteConn) RegisterAggregator(name string, impl interface{}, pure bool
ai.stepArgConverters = append(ai.stepArgConverters, conv)
}
if step.IsVariadic() {
conv, err := callbackArg(t.In(start + stepNArgs).Elem())
conv, err := callbackArg(step.In(start + stepNArgs).Elem())
if err != nil {
return err
}
......@@ -740,6 +740,8 @@ func (c *SQLiteConn) RegisterAggregator(name string, impl interface{}, pure bool
// AutoCommit return which currently auto commit or not.
func (c *SQLiteConn) AutoCommit() bool {
c.mu.Lock()
defer c.mu.Unlock()
return int(C.sqlite3_get_autocommit(c.db)) != 0
}
......@@ -1340,6 +1342,9 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
mutex|C.SQLITE_OPEN_READWRITE|C.SQLITE_OPEN_CREATE,
nil)
if rv != 0 {
if db != nil {
C.sqlite3_close_v2(db)
}
return nil, Error{Code: ErrNo(rv)}
}
if db == nil {
......@@ -1376,7 +1381,7 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
// - Activate User Authentication
// Check if the user wants to activate User Authentication.
// If so then first create a temporary AuthConn to the database
// This is possible because we are already succesfully authenticated.
// This is possible because we are already successfully authenticated.
//
// - Check if `sqlite_user`` table exists
// YES => Add the provided user from DSN as Admin User and
......@@ -1387,7 +1392,7 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
// Create connection to SQLite
conn := &SQLiteConn{db: db, loc: loc, txlock: txlock}
// Password Cipher has to be registerd before authentication
// Password Cipher has to be registered before authentication
if len(authCrypt) > 0 {
switch strings.ToUpper(authCrypt) {
case "SHA1":
......@@ -1674,7 +1679,7 @@ func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, er
defer C.free(unsafe.Pointer(pquery))
var s *C.sqlite3_stmt
var tail *C.char
rv := C._sqlite3_prepare_v2_internal(c.db, pquery, -1, &s, &tail)
rv := C._sqlite3_prepare_v2_internal(c.db, pquery, C.int(-1), &s, &tail)
if rv != C.SQLITE_OK {
return nil, c.lastError()
}
......@@ -1718,7 +1723,7 @@ func (c *SQLiteConn) GetFilename(schemaName string) string {
// GetLimit returns the current value of a run-time limit.
// See: sqlite3_limit, http://www.sqlite.org/c3ref/limit.html
func (c *SQLiteConn) GetLimit(id int) int {
return int(C._sqlite3_limit(c.db, C.int(id), -1))
return int(C._sqlite3_limit(c.db, C.int(id), C.int(-1)))
}
// SetLimit changes the value of a run-time limits.
......@@ -2024,16 +2029,11 @@ func (rc *SQLiteRows) Next(dest []driver.Value) error {
case C.SQLITE_BLOB:
p := C.sqlite3_column_blob(rc.s.s, C.int(i))
if p == nil {
dest[i] = nil
dest[i] = []byte{}
continue
}
n := int(C.sqlite3_column_bytes(rc.s.s, C.int(i)))
switch dest[i].(type) {
default:
slice := make([]byte, n)
copy(slice[:], (*[1 << 30]byte)(p)[0:n])
dest[i] = slice
}
n := C.sqlite3_column_bytes(rc.s.s, C.int(i))
dest[i] = C.GoBytes(p, n)
case C.SQLITE_NULL:
dest[i] = nil
case C.SQLITE_TEXT:
......@@ -2062,9 +2062,8 @@ func (rc *SQLiteRows) Next(dest []driver.Value) error {
}
dest[i] = t
default:
dest[i] = []byte(s)
dest[i] = s
}
}
}
return nil
......
......@@ -13,7 +13,7 @@ import (
// This file provides several different implementations for the
// default embedded sqlite_crypt function.
// This function is uses a ceasar-cypher by default
// This function is uses a caesar-cypher by default
// and is used within the UserAuthentication module to encode
// the password.
//
......@@ -40,7 +40,7 @@ import (
// password X, sqlite_crypt(X,NULL) is run. A new random salt is selected
// when the second argument is NULL.
//
// The built-in version of of sqlite_crypt() uses a simple Ceasar-cypher
// The built-in version of of sqlite_crypt() uses a simple Caesar-cypher
// which prevents passwords from being revealed by searching the raw database
// for ASCII text, but is otherwise trivally broken. For better password
// security, the database should be encrypted using the SQLite Encryption
......
......@@ -10,7 +10,6 @@ package sqlite3
import (
"database/sql/driver"
"errors"
"context"
)
......@@ -18,7 +17,8 @@ import (
// Ping implement Pinger.
func (c *SQLiteConn) Ping(ctx context.Context) error {
if c.db == nil {
return errors.New("Connection was closed")
// must be ErrBadConn for sql to close the database
return driver.ErrBadConn
}
return nil
}
......
......@@ -311,12 +311,18 @@ struct sqlite3_api_routines {
int (*str_errcode)(sqlite3_str*);
int (*str_length)(sqlite3_str*);
char *(*str_value)(sqlite3_str*);
/* Version 3.25.0 and later */
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*),
void (*xValue)(sqlite3_context*),
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
void(*xDestroy)(void*));
/* Version 3.26.0 and later */
const char *(*normalized_sql)(sqlite3_stmt*);
/* Version 3.28.0 and later */
int (*stmt_isexplain)(sqlite3_stmt*);
int (*value_frombind)(sqlite3_value*);
};
/*
......@@ -604,6 +610,11 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_str_value sqlite3_api->str_value
/* Version 3.25.0 and later */
#define sqlite3_create_window_function sqlite3_api->create_window_function
/* Version 3.26.0 and later */
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
/* Version 3.28.0 and later */
#define sqlite3_stmt_isexplain sqlite3_api->isexplain
#define sqlite3_value_frombind sqlite3_api->frombind
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
......
......@@ -129,7 +129,7 @@ github.com/blang/semver
github.com/bronze1man/goStrongswanVici
# github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23
github.com/buger/jsonparser
# github.com/canonical/go-dqlite v1.1.0
# github.com/canonical/go-dqlite v1.1.0 => github.com/rancher/go-dqlite v1.1.0-k3s.1
github.com/canonical/go-dqlite/client
github.com/canonical/go-dqlite
github.com/canonical/go-dqlite/internal/logging
......@@ -660,7 +660,7 @@ github.com/mailru/easyjson/buffer
github.com/matryer/moq/pkg/moq
# github.com/mattn/go-shellwords v1.0.5
github.com/mattn/go-shellwords
# github.com/mattn/go-sqlite3 v1.10.0
# github.com/mattn/go-sqlite3 v1.11.0
github.com/mattn/go-sqlite3
# github.com/matttproud/golang_protobuf_extensions v1.0.1
github.com/matttproud/golang_protobuf_extensions/pbutil
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment