From 9f830bbee6ff0494372ba701d051c039cc5a7fd6 Mon Sep 17 00:00:00 2001 From: Alec Warner Date: Mon, 3 Dec 2018 22:44:29 -0500 Subject: Initial commit of rsync docker stuff. Signed-off-by: Alec Warner --- foundation.gentoo.org/golang/members/Makefile | 5 -- foundation.gentoo.org/golang/members/list.go | 33 -------- foundation.gentoo.org/golang/members/main.go | 22 ----- foundation.gentoo.org/golang/members/member.proto | 16 ---- sites/foundation_tracker/Makefile | 5 -- sites/foundation_tracker/member.proto | 16 ---- .../golang/members/data/member.pb.go | 97 +++++++++++++++++++--- .../golang/members/data/member.proto | 28 ++++++- .../golang/members/importers/ldap.go | 10 +-- src/foundation.gentoo.org/golang/members/main.go | 22 ----- .../golang/members/maintenance/memberroll.go | 79 +++++++++++++++--- src/infra.gentoo.org/rsync-node/Dockerfile | 21 +++++ src/infra.gentoo.org/rsync-node/rsyncd.conf | 19 +++++ src/infra.gentoo.org/rsync-node/wrap_rsync.sh | 43 ++++++++++ 14 files changed, 263 insertions(+), 153 deletions(-) delete mode 100644 foundation.gentoo.org/golang/members/Makefile delete mode 100644 foundation.gentoo.org/golang/members/list.go delete mode 100644 foundation.gentoo.org/golang/members/main.go delete mode 100644 foundation.gentoo.org/golang/members/member.proto delete mode 100644 sites/foundation_tracker/Makefile delete mode 100644 sites/foundation_tracker/member.proto delete mode 100644 src/foundation.gentoo.org/golang/members/main.go create mode 100644 src/infra.gentoo.org/rsync-node/Dockerfile create mode 100644 src/infra.gentoo.org/rsync-node/rsyncd.conf create mode 100755 src/infra.gentoo.org/rsync-node/wrap_rsync.sh diff --git a/foundation.gentoo.org/golang/members/Makefile b/foundation.gentoo.org/golang/members/Makefile deleted file mode 100644 index b050cf8..0000000 --- a/foundation.gentoo.org/golang/members/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -member.pb.go: - protoc --go_out ./ member.proto - -clean: - rm member.pb.go diff --git a/foundation.gentoo.org/golang/members/list.go b/foundation.gentoo.org/golang/members/list.go deleted file mode 100644 index c997f45..0000000 --- a/foundation.gentoo.org/golang/members/list.go +++ /dev/null @@ -1,33 +0,0 @@ -package members - -import ( - "flag" - "fmt" - "os" - "strings" - - "github.com/google/subcommands" - "golang.org/x/net/context" -) - -type listCmd struct { - membersListPath string -} - -func (*listCmd) Name() string { return "list" } -func (*listCmd) Synopsis() string { return "list members to stdout." } -func (*listCmd) Usage() string { - return `print --membersListPath /path/to/members` -} - -func (p *listCmd) SetFlags(f *flag.FlagSet) { - f.StringVar(&p.membersListPath, "membersListPath", './members', "path to members list") -} - -func (p *listCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - for _, arg := range f.Args() { - fmt.Printf("%s ", arg) - } - fmt.Println() - return subcommands.ExitSuccess -} diff --git a/foundation.gentoo.org/golang/members/main.go b/foundation.gentoo.org/golang/members/main.go deleted file mode 100644 index c726053..0000000 --- a/foundation.gentoo.org/golang/members/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package members - -import ( - "flag" - "os" - "strings" - - "github.com/google/subcommands" - "golang.org/x/net/context" -) - - -func main() { - subcommands.Register(subcommands.HelpCommand(), "") - subcommands.Register(subcommands.FlagsCommand(), "") - subcommands.Register(subcommands.CommandsCommand(), "") - subcommands.Register(&list.ListCmd{}, "") - - flag.Parse() - ctx := context.Background() - os.Exit(int(subcommands.Execute(ctx))) -} diff --git a/foundation.gentoo.org/golang/members/member.proto b/foundation.gentoo.org/golang/members/member.proto deleted file mode 100644 index 22969af..0000000 --- a/foundation.gentoo.org/golang/members/member.proto +++ /dev/null @@ -1,16 +0,0 @@ -message Member { - // Unique idenfier for each member. - optional int64 id = 1; - - // Email information - repeated string email = 2; - - // Name - repeated string name = 3; - - // microsecond timestamp of joining. - optional int64 join_timestamp_us = 4; - - // microsecond timestamp of any elections member participated in - repeated int64 voted_in_election = 5; -} diff --git a/sites/foundation_tracker/Makefile b/sites/foundation_tracker/Makefile deleted file mode 100644 index b050cf8..0000000 --- a/sites/foundation_tracker/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -member.pb.go: - protoc --go_out ./ member.proto - -clean: - rm member.pb.go diff --git a/sites/foundation_tracker/member.proto b/sites/foundation_tracker/member.proto deleted file mode 100644 index 22969af..0000000 --- a/sites/foundation_tracker/member.proto +++ /dev/null @@ -1,16 +0,0 @@ -message Member { - // Unique idenfier for each member. - optional int64 id = 1; - - // Email information - repeated string email = 2; - - // Name - repeated string name = 3; - - // microsecond timestamp of joining. - optional int64 join_timestamp_us = 4; - - // microsecond timestamp of any elections member participated in - repeated int64 voted_in_election = 5; -} diff --git a/src/foundation.gentoo.org/golang/members/data/member.pb.go b/src/foundation.gentoo.org/golang/members/data/member.pb.go index 3d39f40..4f12ab6 100644 --- a/src/foundation.gentoo.org/golang/members/data/member.pb.go +++ b/src/foundation.gentoo.org/golang/members/data/member.pb.go @@ -11,6 +11,7 @@ It is generated from these files: It has these top-level messages: MemberRoll Member + Status */ package data @@ -21,6 +22,41 @@ import math "math" var _ = proto.Marshal var _ = math.Inf +type Status_State int32 + +const ( + // Member has voting rights + Status_ACTIVE Status_State = 0 + // Member had voting rights, but is no longer a foundation member. + Status_EMERITUS Status_State = 1 +) + +var Status_State_name = map[int32]string{ + 0: "ACTIVE", + 1: "EMERITUS", +} +var Status_State_value = map[string]int32{ + "ACTIVE": 0, + "EMERITUS": 1, +} + +func (x Status_State) Enum() *Status_State { + p := new(Status_State) + *p = x + return p +} +func (x Status_State) String() string { + return proto.EnumName(Status_State_name, int32(x)) +} +func (x *Status_State) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(Status_State_value, data, "Status_State") + if err != nil { + return err + } + *x = Status_State(value) + return nil +} + type MemberRoll struct { // When we add a new member, they should receive this ID. // We rely on clients to increment this ID during add operations. @@ -55,11 +91,16 @@ type Member struct { Email []string `protobuf:"bytes,2,rep,name=email" json:"email,omitempty"` // Name Name []string `protobuf:"bytes,3,rep,name=name" json:"name,omitempty"` - // microsecond timestamp of joining. - JoinTimestampUs *int64 `protobuf:"varint,4,opt,name=join_timestamp_us" json:"join_timestamp_us,omitempty"` // microsecond timestamp of any elections member participated in - VotedInElection []int64 `protobuf:"varint,5,rep,name=voted_in_election" json:"voted_in_election,omitempty"` - XXX_unrecognized []byte `json:"-"` + VotedInElection []int64 `protobuf:"varint,4,rep,name=voted_in_election" json:"voted_in_election,omitempty"` + // Status's of a given member. Typically most users have 1 state + // (e.g. they are an active member of the foundation.) But we will + // try to record all state changes here, so if people leave and come + // back we have a record. + State []*Status `protobuf:"bytes,5,rep,name=state" json:"state,omitempty"` + // A list of gpg key bytes for each member. + Gpgkey [][]byte `protobuf:"bytes,6,rep,name=gpgkey" json:"gpgkey,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *Member) Reset() { *m = Member{} } @@ -87,19 +128,55 @@ func (m *Member) GetName() []string { return nil } -func (m *Member) GetJoinTimestampUs() int64 { - if m != nil && m.JoinTimestampUs != nil { - return *m.JoinTimestampUs +func (m *Member) GetVotedInElection() []int64 { + if m != nil { + return m.VotedInElection } - return 0 + return nil } -func (m *Member) GetVotedInElection() []int64 { +func (m *Member) GetState() []*Status { if m != nil { - return m.VotedInElection + return m.State } return nil } +func (m *Member) GetGpgkey() [][]byte { + if m != nil { + return m.Gpgkey + } + return nil +} + +type Status struct { + // State of the member. + State *Status_State `protobuf:"varint,1,opt,name=state,enum=data.Status_State,def=0" json:"state,omitempty"` + // Time when state change occured. + StatusChange *int64 `protobuf:"varint,2,opt,name=status_change" json:"status_change,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Status) Reset() { *m = Status{} } +func (m *Status) String() string { return proto.CompactTextString(m) } +func (*Status) ProtoMessage() {} + +const Default_Status_State Status_State = Status_ACTIVE + +func (m *Status) GetState() Status_State { + if m != nil && m.State != nil { + return *m.State + } + return Default_Status_State +} + +func (m *Status) GetStatusChange() int64 { + if m != nil && m.StatusChange != nil { + return *m.StatusChange + } + return 0 +} + func init() { + proto.RegisterEnum("data.Status_State", Status_State_name, Status_State_value) } diff --git a/src/foundation.gentoo.org/golang/members/data/member.proto b/src/foundation.gentoo.org/golang/members/data/member.proto index 2e35208..500364e 100644 --- a/src/foundation.gentoo.org/golang/members/data/member.proto +++ b/src/foundation.gentoo.org/golang/members/data/member.proto @@ -18,9 +18,29 @@ message Member { // Name repeated string name = 3; - // microsecond timestamp of joining. - optional int64 join_timestamp_us = 4; - // microsecond timestamp of any elections member participated in - repeated int64 voted_in_election = 5; + repeated int64 voted_in_election = 4; + + // Status's of a given member. Typically most users have 1 state + // (e.g. they are an active member of the foundation.) But we will + // try to record all state changes here, so if people leave and come + // back we have a record. + repeated Status state = 5; + + // A list of gpg key bytes for each member. + repeated bytes gpgkey = 6; +} + +message Status { + enum State { + // Member has voting rights + ACTIVE = 0; + // Member had voting rights, but is no longer a foundation member. + EMERITUS = 1; + } + + // State of the member. + optional State state = 1 [default=ACTIVE]; + // Time when state change occured. + optional int64 status_change = 2; } diff --git a/src/foundation.gentoo.org/golang/members/importers/ldap.go b/src/foundation.gentoo.org/golang/members/importers/ldap.go index 02db9b0..02cc7eb 100644 --- a/src/foundation.gentoo.org/golang/members/importers/ldap.go +++ b/src/foundation.gentoo.org/golang/members/importers/ldap.go @@ -1,9 +1,3 @@ -package ldap +package importers -import ( - "gopkg.in/ldap.v2" -) - -func listDevs(ldap.v2.Conn) { - -} +import () diff --git a/src/foundation.gentoo.org/golang/members/main.go b/src/foundation.gentoo.org/golang/members/main.go deleted file mode 100644 index 18e7f65..0000000 --- a/src/foundation.gentoo.org/golang/members/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "flag" - "os" - - "foundation.gentoo.org/golang/members/maintenance" - "github.com/google/subcommands" - "golang.org/x/net/context" -) - - -func main() { - subcommands.Register(subcommands.HelpCommand(), "") - subcommands.Register(subcommands.FlagsCommand(), "") - subcommands.Register(subcommands.CommandsCommand(), "") - subcommands.Register(&maintenance.ListCmd{}, "") - - flag.Parse() - ctx := context.Background() - os.Exit(int(subcommands.Execute(ctx))) -} diff --git a/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go b/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go index 05017ea..12369d5 100644 --- a/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go +++ b/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go @@ -1,20 +1,22 @@ package maintenance import ( - "io/ioutil" + "fmt" + "io/ioutil" + "time" - "github.com/golang/protobuf/proto" - pb "foundation.gentoo.org/golang/members/data" + pb "foundation.gentoo.org/golang/members/data" + "github.com/golang/protobuf/proto" ) // Struct MemberRoll defines operations on a MemberRoll // and holds the underlying data. type MemberRoll struct { - roll *pb.MemberRoll + roll *pb.MemberRoll } // Create a new memberRoll from a file. -func NewMemberRoll (path string) (*MemberRoll, error) { +func NewMemberRoll(path string) (*MemberRoll, error) { in, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -30,10 +32,10 @@ func NewMemberRoll (path string) (*MemberRoll, error) { } // NewEmptyMemberRoll creates an empty MemberRoll -func NewEmptyMemberRoll() (*MemberRoll) { +func NewEmptyMemberRoll() *MemberRoll { // The first member gets ID 1. return &MemberRoll{ - roll: &pb.MemberRoll { + roll: &pb.MemberRoll{ NextMemberId: proto.Int64(1), }, } @@ -42,18 +44,71 @@ func NewEmptyMemberRoll() (*MemberRoll) { // Save will write the MemberRoll to file at path. func (mr *MemberRoll) Save(path string) (bool, error) { bytes, err := proto.Marshal(mr.roll) - if err != nil { return false, err } + if err != nil { + return false, err + } if err = ioutil.WriteFile(path, bytes, 0644); err != nil { return false, err } return true, nil } +// Does this email match an existing member? +func (mr *MemberRoll) MatchEmail(email string) *pb.Member { + member := &pb.Member{ + Email: []string{email}, + } + return mr.Match(member) +} + +// Members match if they shared an email address. +func Match(m1, m2 *pb.Member) bool { + for _, i := range m1.GetEmail() { + for _, j := range m2.GetEmail() { + if i == j { + return true + } + } + } + return false +} + +// Match will try to find if m is in mr. +func (mr *MemberRoll) Match(m *pb.Member) *pb.Member { + for _, i := range mr.roll.GetMembers() { + if Match(m, i) { + return i + } + } + return nil +} + +// low-level function for add raw member protobufs to the roll. +func (mr *MemberRoll) Add(m pb.Member) (bool, error) { + match := mr.Match(&m) + if match != nil { + err := fmt.Errorf("Adding existing member: %v", match) + return false, err + } + mr.roll.Members = append(mr.roll.Members, &m) + return true, nil +} + +func (mr *MemberRoll) RecordVote(id int64, t time.Time) error { + return nil +} + +func (mr *MemberRoll) Print() { + for _, member := range mr.roll.GetMembers() { + fmt.Printf("Member: %v\n", member) + } +} + // GetMaxMemberId computes the largest ID amongst a memberroll. // This works by assuming members are not removed from a roll. -func (mr *MemberRoll) GetMaxMemberId() (int64) { - var id int64 = 0; - for _, member := range (mr.roll.GetMembers()) { +func (mr *MemberRoll) GetMaxMemberId() int64 { + var id int64 = 0 + for _, member := range mr.roll.GetMembers() { member_id := member.GetId() if member_id > id { id = member_id @@ -62,7 +117,7 @@ func (mr *MemberRoll) GetMaxMemberId() (int64) { return id } -func (mr *MemberRoll) GetNextMemberId() (int64) { +func (mr *MemberRoll) GetNextMemberId() int64 { id := mr.GetNextMemberId() // If the member roll lost its ID (not a required field.) // Recalculate based on existing membership. diff --git a/src/infra.gentoo.org/rsync-node/Dockerfile b/src/infra.gentoo.org/rsync-node/Dockerfile new file mode 100644 index 0000000..3530c25 --- /dev/null +++ b/src/infra.gentoo.org/rsync-node/Dockerfile @@ -0,0 +1,21 @@ +# image is based on stage3-amd64 +FROM gentoo/stage3-amd64:latest +COPY rsyncd.conf /opt/rsync/rsyncd.conf +COPY wrap_rsync.sh /opt/rsync/wrap_rsync.sh + +# Normally I would advocate for ARG here and pass arguments to wrap_rsync. +# This would enable new docker builds with arguments like: +# docker build . --build_arg WAIT_TIME=30m -t gentoo/rsync +# However, ARG's cannot be passed to ENTRYPOINTs, so we set these as ENV instead. +# Mirror to get data from. +ENV SOURCE_MIRROR=rsync://boobie.gentoo.org/gentoo-portage +# ENV SOURCE_MIRROR=rsync://rsync.us.gentoo.org/gentoo-portage +# Where to write the data in this container. +ENV DEST_DIR=/dev/shm/gentoo +# How long to wait between syncs; must be a valid argument to sleep +ENV WAIT_TIME=10m + +# Expose Rsync port +EXPOSE 873 + +ENTRYPOINT /opt/rsync/wrap_rsync.sh diff --git a/src/infra.gentoo.org/rsync-node/rsyncd.conf b/src/infra.gentoo.org/rsync-node/rsyncd.conf new file mode 100644 index 0000000..7fbd4b9 --- /dev/null +++ b/src/infra.gentoo.org/rsync-node/rsyncd.conf @@ -0,0 +1,19 @@ + +uid = nobody +gid = nobody +# We are in a container, who cares. +# use chroot = no +# Let clients use as much as they want; CPU control is in a load balancer in front of us. +max connections = 0 + +motd file = ./rsyncd.motd +log file = /var/log/rsync.log +transfer logging = yes +log format = %t %a %m %f %b +syslog facility = local3 +timeout = 300 + +[gentoo-portage] +path = /dev/shm/gentoo +comment = Gentoo Linux Portage tree mirror +exclude = distfiles diff --git a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh new file mode 100755 index 0000000..f92934c --- /dev/null +++ b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# On container start, run an rsync to get a good copy of the tree. +# Then execute rsyncd; we will start serving once the sync completes. +# Then keep syncing in the background every 30m. + +function sync() { + OPTS=( + --quiet + --recursive + --links + --perms + --times + --delete + --timeout=300 + --checksum + ) + SRC="${2}" + DST="${1}" + + echo "Started update at" $(date) >> $0.log 2>&1 + logger -t rsync "re-rsyncing the gentoo-portage tree" + /usr/bin/rsync ${OPTS[@]} "${SRC}" "${DST}" >> $0.log 2>&1 + echo "End: "$(date) >> $0.log 2>&1 +} + +sync "${DEST_DIR}/serving" "${SOURCE_MIRROR}" # this is synchronous. + +# Then launch rsyncd; it will detach into the background. +rsync --daemon --config="/opt/rsync/rsyncd.conf" + +while true +do + sleep "${WAIT_TIME}" + tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX) + # If we fail to sync, just try again. + if ! sync "${tmp}" "${SOURCE_MIRROR}"; then + rm -rf "${tmp}" + continue + fi + # Atomically rename + mv -f "${tmp}" "${DEST_DIR}/serving" +done -- cgit v1.2.3-65-gdbad