changed xlget docker, installer options and add import of ipv4 in hos…

…ts format

Dockerfile.xlget

@@ -11,6 +11,12 @@ ENV github_token=$github_token
 # Install git and certificates
 RUN apk update && apk add --no-cache git make ca-certificates && update-ca-certificates
 
+# create user for service
+RUN adduser -D -g 'luids' luxlist \
+    && mkdir -p /var/lib/luids/xlist \
+    && touch /var/lib/luids/xlist/.keep \
+    && chown -R luxlist /var/lib/luids/xlist
+
 WORKDIR /app
 
 ## config git
@@ -28,24 +34,18 @@ COPY . .
 RUN make binaries SYSTEM="GOOS=linux GOARCH=${arch}"
 
 ## create docker
-FROM alpine
+FROM scratch
 
 LABEL maintainer="Luis Guillén Civera <luisguillenc@gmail.com>"
 
-RUN apk update && apk add --no-cache ca-certificates && update-ca-certificates
-
-# create user for service
-RUN adduser -D -g 'luids' luxlist && \
-	mkdir -p /var/lib/luids/xlist && \
-	chown luxlist /var/lib/luids/xlist && \
-	mkdir -p /var/cache/luids/xlist && \
-	chown luxlist /var/cache/luids/xlist
+# Import the user and group files from the builder.
+COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build-env /etc/passwd /etc/passwd
 
 COPY --from=build-env /app/bin/xlget /bin/
-COPY --from=build-env /app/configs/docker/xlget/* /etc/luids/xlist/
+COPY --from=build-env /app/configs/docker/xlistd/*  /etc/luids/xlist/
+COPY --from=build-env /var/lib/luids /var/lib/luids
 
 USER luxlist
-
 VOLUME [ "/etc/luids", "/var/lib/luids/xlist" ]
-
 CMD [ "/bin/xlget", "--auto", "--config", "/etc/luids/xlist/xlget.toml" ]

cmd/xlistd/depend.go

@@ -37,7 +37,7 @@ func createHealthSrv(msrv *serverd.Manager, logger yalogi.Logger) error {
 			return err
 		}
 		msrv.Register(serverd.Service{
-			Name:     "health.server",
+			Name:     fmt.Sprintf("health.[%s]", cfgHealth.ListenURI),
 			Start:    func() error { go health.Serve(hlis); return nil },
 			Shutdown: func() { health.Close() },
 		})
@@ -96,7 +96,7 @@ func createLists(apisvc apiservice.Discover, msrv *serverd.Manager, logger yalog
 		return nil, err
 	}
 	msrv.Register(serverd.Service{
-		Name:     "xlist",
+		Name:     "xlist.service",
 		Start:    builder.Start,
 		Shutdown: func() { builder.Shutdown() },
 	})

deployments/installer/installer.sh

@@ -403,55 +403,13 @@ create_service_config() {
 	if [ ! -f $ETC_DIR/$NAME/xlistd.toml ]; then
 		log "creating $ETC_DIR/$NAME/xlistd.toml"
 		{ cat > $ETC_DIR/$NAME/xlistd.toml <<EOF
-######################
-## Service settings ##
-######################
 [xlist]
 certsdir   = "${ETC_DIR}/ssl"
-sourcesdir = "${VAR_DIR}/${NAME}"
+datadir    = "${VAR_DIR}/${NAME}"
 
-[xlist.db]
+[xlist.service]
 files      = [ "${ETC_DIR}/${NAME}/services.json" ]
-#dirs       = [ "${ETC_DIR}/${NAME}/services.d" ]
-
-#[xlist.dnsxl]
-#resolvers  = [ "8.8.8.8", "8.8.4.4" ]
-#resolvconf = false
-
-#[xlist.api.check]
-#rootid     = "root"
-#disclosure = false
-#exposeping = false
-
-#####################
-## Server settings ##
-#####################
-## By default only serves grpc API and listen in localhost
-#[server]
-#listenuri  = "tcp://0.0.0.0:5801"
-#certca     = "${ETC_DIR}/ssl/certs/CA.crt"
-#certfile   = "${ETC_DIR}/ssl/certs/server.crt"
-#keyfile    = "${ETC_DIR}/ssl/private/server.key"
-#clientauth = false
-#allowed    = [ "127.0.0.1", "192.168.0.0/16" ]
-#metrics    = false
-
-#####################
-## Health settings ##
-#####################
-## By default health service is disabled
-## uncoment this for metrics and monitoring
-#[health]
-#listenuri  = "tcp://127.0.0.1:8081"
-#metrics    = true
-#allowed    = [ "127.0.0.1" ]
-
-##################
-## Log settings ##
-##################
-[log]
-format  = "log"
-level   = "info"
+
 EOF
 		} &>>$LOG_FILE
 		[ $? -ne 0 ] && step_err && return 1
@@ -471,10 +429,12 @@ EOF
 		log "creating $ETC_DIR/$NAME/xlget.toml"
 		{ cat > $ETC_DIR/$NAME/xlget.toml <<EOF
 [xlget]
+outputdir = "${VAR_DIR}/${NAME}"
+cachedir  = "${CACHE_DIR}/${NAME}"
+
+[xlget.source]
 files     = [ "${ETC_DIR}/${NAME}/sources.json" ]
-#dirs      = [ "${ETC_DIR}/${NAME}/sources.d" ]
-output    = "${VAR_DIR}/${NAME}"
-cache     = "${CACHE_DIR}/${NAME}"
+
 EOF
 		} &>>$LOG_FILE
 		[ $? -ne 0 ] && step_err && return 1

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/cavaliercoder/grab v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/luids-io/api v0.0.0-20200804060453-90be0601205f
+	github.com/luids-io/api v0.0.0-20200805154000-e6a3b4e0bfac
 	github.com/luids-io/common v0.0.0-20200510171107-c796758176cb
 	github.com/luids-io/core v0.0.0-20200729141657-1a2e3655d06f
 	github.com/miekg/dns v1.1.29

go.sum

@@ -175,6 +175,8 @@ github.com/luids-io/api v0.0.0-20200803144013-d504cb3dc97d h1:rwyEkPZOg+FrZDkGCA
 github.com/luids-io/api v0.0.0-20200803144013-d504cb3dc97d/go.mod h1:4IkMQVmc9BMnlA7YXTYpSwDn9+Y+C51cwXEdLqe5SVA=
 github.com/luids-io/api v0.0.0-20200804060453-90be0601205f h1:ubJsaYBFGu1XtOpPsUMyNjEo3xM4K9ayiHwjemdRSBE=
 github.com/luids-io/api v0.0.0-20200804060453-90be0601205f/go.mod h1:4IkMQVmc9BMnlA7YXTYpSwDn9+Y+C51cwXEdLqe5SVA=
+github.com/luids-io/api v0.0.0-20200805154000-e6a3b4e0bfac h1:a+J8rs98AVNEqb1p7qjJQUMmqvqIyL3flnnoSigP9Bc=
+github.com/luids-io/api v0.0.0-20200805154000-e6a3b4e0bfac/go.mod h1:4IkMQVmc9BMnlA7YXTYpSwDn9+Y+C51cwXEdLqe5SVA=
 github.com/luids-io/common v0.0.0-20200510171107-c796758176cb h1:Huym8yYsvK0qKgWFti9FTydmPUONAXlMSZZZPkJiqbY=
 github.com/luids-io/common v0.0.0-20200510171107-c796758176cb/go.mod h1:C0Mh01EMI67r4pgimVfxI60IrReIGx8cfWZi9dSq1pE=
 github.com/luids-io/core v0.0.0-20200529043026-f84c3788ea2e h1:mqP6LxY5lBr/FJXbT2HlewymlLXJWYKSujmec+AdXDo=

pkg/xlget/do_converthosts.go

@@ -85,8 +85,23 @@ func (p HostsConv) Convert(ctx context.Context, in io.Reader, out io.Writer) (ma
 			}
 		}
 		if len(fields) > 1 {
+			added := false
 			_, isDefault := hostDefaults[fields[1]]
-			if p.checks(xlist.Domain) && xlist.ValidResource(fields[1], xlist.Domain) {
+			if p.checks(xlist.IPv4) && xlist.ValidResource(fields[1], xlist.IPv4) {
+				if !isDefault || (isDefault && p.WithDefaults) {
+					account[xlist.IPv4] = account[xlist.IPv4] + 1
+					fmt.Fprintf(out, "ip4,plain,%s\n", fields[1])
+					added = true
+				}
+			}
+			if p.checks(xlist.IPv6) && xlist.ValidResource(fields[1], xlist.IPv6) && !added {
+				if !isDefault || (isDefault && p.WithDefaults) {
+					account[xlist.IPv6] = account[xlist.IPv6] + 1
+					fmt.Fprintf(out, "ip6,plain,%s\n", fields[1])
+					added = true
+				}
+			}
+			if p.checks(xlist.Domain) && xlist.ValidResource(fields[1], xlist.Domain) && !added {
 				if !isDefault || (isDefault && p.WithDefaults) {
 					// apply opts
 					if p.Opts.MinDomain > 0 {