diff --git a/api.go b/api.go index dd57abc4..1b4f00ae 100644 --- a/api.go +++ b/api.go @@ -31,10 +31,10 @@ import ( func APICommand() *cobra.Command { roles := []string{} usernamePrefix := "" - var allowEmptyBillingEntity bool + var allowEmptyBillingEntity, skipBillingEntityValidation bool ob := &odooStorageBuilder{} - ost := orgStore.New(&roles, &usernamePrefix, &allowEmptyBillingEntity) + ost := orgStore.New(&roles, &usernamePrefix, &allowEmptyBillingEntity, &skipBillingEntityValidation) ib := &invitationStorageBuilder{usernamePrefix: &usernamePrefix} cmd, err := builder.APIServer. @@ -55,6 +55,7 @@ func APICommand() *cobra.Command { cmd.Flags().StringSliceVar(&roles, "cluster-roles", []string{}, "Cluster Roles to bind when creating an organization") cmd.Flags().StringVar(&usernamePrefix, "username-prefix", "", "Prefix prepended to username claims. Usually the same as \"--oidc-username-prefix\" of the Kubernetes API server") cmd.Flags().BoolVar(&allowEmptyBillingEntity, "allow-empty-billing-entity", true, "Allow empty billing entity references") + cmd.Flags().BoolVar(&skipBillingEntityValidation, "organization-skip-billing-entity-validation", false, "Skip validation of billing entity references") cmd.Flags().StringVar(&ob.billingEntityStorage, "billing-entity-storage", "fake", "Storage backend for billing entities. Supported values: fake, odoo8, odoo16") diff --git a/apiserver/billing/odoostorage/odoo/odoo16/odoo16.go b/apiserver/billing/odoostorage/odoo/odoo16/odoo16.go index 8dafd530..fa327abc 100644 --- a/apiserver/billing/odoostorage/odoo/odoo16/odoo16.go +++ b/apiserver/billing/odoostorage/odoo/odoo16/odoo16.go @@ -23,15 +23,16 @@ import ( const VSHNAccountingContactNameKey = "billing.appuio.io/vshn-accounting-contact-name" // Used to identify the accounting contact of a company. -const roleAccountCategory = 70 -const companyCategory = 2 const invoiceType = "invoice" +// TODO(bastjan) test if still needed in odoo16 +const companyCategory = 2 + // Used to generate the UUID for the .metadata.uid field. var metaUIDNamespace = uuid.MustParse("7550b1ae-7a2a-485e-a75d-6f931b2cd73f") -var roleAccountFilter = odooclient.NewCriterion("category_id", "in", []int{roleAccountCategory}) var activeFilter = odooclient.NewCriterion("active", "=", true) +var invoiceTypeFilter = odooclient.NewCriterion("type", "=", invoiceType) var notInflightFilter = odooclient.NewCriterion("vshn_control_api_inflight", "=", false) var mustInflightFilter = odooclient.NewCriterion("vshn_control_api_inflight", "!=", false) @@ -97,8 +98,8 @@ type FailedRecordScrubber struct { sessionCreator func(ctx context.Context) (Odoo16Client, error) } +//go:generate go run go.uber.org/mock/mockgen -destination=./odoo16mock/$GOFILE -package odoo16mock . Odoo16Client type Odoo16Client interface { - Read(string, []int64, *odooclient.Options, interface{}) error Update(string, []int64, interface{}) error FindResPartners(*odooclient.Criteria, *odooclient.Options) (*odooclient.ResPartners, error) CreateResPartner(*odooclient.ResPartner) (int64, error) @@ -127,28 +128,45 @@ func (s *Odoo16Storage) get(ctx context.Context, name string) (company odooclien return odooclient.ResPartner{}, odooclient.ResPartner{}, err } - u := []odooclient.ResPartner{} - err = session.Read(odooclient.ResPartnerModel, []int64{int64(id)}, fetchPartnerFieldOpts, &u) + accp, err := session.FindResPartners( + newValidInvoiceRecordCriteria().AddCriterion(odooclient.NewCriterion("id", "=", id)), + fetchPartnerFieldOpts) if err != nil { - return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("fetching accounting contact by ID: %w", err) + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("error fetching accounting contact %d: %w", id, err) + } + if accp == nil { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("fetching accounting contact %d returned nil", id) + } + acc := *accp + if len(acc) <= 0 { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("no results when fetching accounting contact %d", id) } - if len(u) <= 0 { - return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("no results when fetching accounting contact by ID") + if len(acc) > 1 { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("multiple results when fetching accounting contact %d", id) } - accountingContact = u[0] + accountingContact = acc[0] if accountingContact.ParentId == nil { return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("accounting contact %d has no parent", id) } - err = session.Read(odooclient.ResPartnerModel, []int64{accountingContact.ParentId.ID}, fetchPartnerFieldOpts, &u) + cpp, err := session.FindResPartners( + odooclient.NewCriteria().AddCriterion(activeFilter).AddCriterion(odooclient.NewCriterion("id", "=", id)), + fetchPartnerFieldOpts) if err != nil { return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("fetching parent %d of accounting contact %d failed: %w", accountingContact.ParentId.ID, id, err) } - if len(u) <= 0 { - return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("no results when fetching parent %d of accounting contact %d failed", accountingContact.ParentId.ID, id) + if cpp == nil { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("fetching parent %d of accounting contact %d returned nil", accountingContact.ParentId.ID, id) } - company = u[0] + cp := *cpp + if len(cp) <= 0 { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("no results when fetching parent %d of accounting contact %d", accountingContact.ParentId.ID, id) + } + if len(cp) > 1 { + return odooclient.ResPartner{}, odooclient.ResPartner{}, fmt.Errorf("multiple results when fetching parent %d of accounting contact %d", accountingContact.ParentId.ID, id) + } + company = cp[0] return company, accountingContact, nil } @@ -161,8 +179,7 @@ func (s *Odoo16Storage) List(ctx context.Context) ([]billingv1.BillingEntity, er return nil, err } - criteria := odooclient.NewCriteria().AddCriterion(roleAccountFilter).AddCriterion(activeFilter).AddCriterion(notInflightFilter) - accPartners, err := session.FindResPartners(criteria, fetchPartnerFieldOpts) + accPartners, err := session.FindResPartners(newValidInvoiceRecordCriteria(), fetchPartnerFieldOpts) if err != nil { return nil, err } @@ -176,7 +193,7 @@ func (s *Odoo16Storage) List(ctx context.Context) ([]billingv1.BillingEntity, er companyIDs = append(companyIDs, int(p.ParentId.ID)) } - criteria = odooclient.NewCriteria().AddCriterion(activeFilter).AddCriterion(odooclient.NewCriterion("id", "in", companyIDs)) + criteria := odooclient.NewCriteria().AddCriterion(activeFilter).AddCriterion(odooclient.NewCriterion("id", "in", companyIDs)) companies, err := session.FindResPartners(criteria, fetchPartnerFieldOpts) if err != nil { return nil, err @@ -426,7 +443,6 @@ func mapBillingEntityToPartners(be billingv1.BillingEntity, countryIDs map[strin func setStaticAccountingContactFields(conf Config, a *odooclient.ResPartner) { a.CategoryId = odooclient.NewRelation() - a.CategoryId.AddRecord(int64(roleAccountCategory)) a.Lang = odooclient.NewSelection(conf.LanguagePreference) a.Type = odooclient.NewSelection(invoiceType) a.PropertyPaymentTermId = odooclient.NewMany2One(int64(conf.PaymentTermID), "") @@ -449,3 +465,10 @@ func splitCommaSeparated(s string) []string { } return p } + +func newValidInvoiceRecordCriteria() *odooclient.Criteria { + return odooclient.NewCriteria(). + AddCriterion(invoiceTypeFilter). + AddCriterion(activeFilter). + AddCriterion(notInflightFilter) +} diff --git a/apiserver/billing/odoostorage/odoo/odoo16/odoo16_test.go b/apiserver/billing/odoostorage/odoo/odoo16/odoo16_test.go index a3572d31..f3a3d093 100644 --- a/apiserver/billing/odoostorage/odoo/odoo16/odoo16_test.go +++ b/apiserver/billing/odoostorage/odoo/odoo16/odoo16_test.go @@ -25,17 +25,17 @@ func TestGet(t *testing.T) { statusTime := st.Local() gomock.InOrder( - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(456), CreateDate: odooclient.NewTime(tn), ParentId: odooclient.NewMany2One(123, ""), Email: odooclient.NewString("accounting@test.com, notifications@test.com"), VshnControlApiMetaStatus: odooclient.NewString("{\"conditions\":[{\"type\":\"ConditionFoo\",\"status\":\"False\",\"lastTransitionTime\":\"" + statusTime.Format(time.RFC3339) + "\",\"reason\":\"Whatever\",\"message\":\"Hello World\"}]}"), - }}).Return(nil), - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + }}, nil), + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(123), Name: odooclient.NewString("Test Company"), - }}).Return(nil), + }}, nil), ) s, err := subject.Get(context.Background(), "be-456") @@ -85,11 +85,10 @@ func TestGetNoParent(t *testing.T) { defer ctrl.Finish() gomock.InOrder( - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(456), Name: odooclient.NewString("Accounting"), - }}, - ).Return(nil), + }}, nil), ) _, err := subject.Get(context.Background(), "be-456") @@ -101,12 +100,12 @@ func TestGet_ParentCantBeRetrieved(t *testing.T) { defer ctrl.Finish() gomock.InOrder( - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(456), Name: odooclient.NewString("Accounting"), ParentId: odooclient.NewMany2One(123, ""), - }}).Return(nil), - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("No such record")), + }}, nil), + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(nil, errors.New("No such record")), ) _, err := subject.Get(context.Background(), "be-456") @@ -189,17 +188,17 @@ func TestCreate(t *testing.T) { // Reset inflight flag mock.EXPECT().Update(odooclient.ResPartnerModel, gomock.InAnyOrder([]int64{700, 702}), gomock.Any()), // Fetch created company - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(702), Name: odooclient.NewString("Max Foobar"), CreateDate: odooclient.NewTime(tn), ParentId: odooclient.NewMany2One(700, ""), Email: odooclient.NewString("accounting@test.com, notifications@test.com"), - }}), - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + }}, nil), + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(700), Name: odooclient.NewString("Test Company"), - }}), + }}, nil), ) s := &billingv1.BillingEntity{ @@ -239,30 +238,30 @@ func TestUpdate(t *testing.T) { gomock.InOrder( // Fetch existing company - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(702), ParentId: odooclient.NewMany2One(700, ""), - }}), - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + }}, nil), + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(700), Name: odooclient.NewString("Test Company"), - }}), + }}, nil), // Update company mock.EXPECT().UpdateResPartner(gomock.Any()), // Update accounting contact mock.EXPECT().UpdateResPartner(gomock.Any()), // Fetch created company - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(702), CreateDate: odooclient.NewTime(tn), ParentId: odooclient.NewMany2One(700, ""), Email: odooclient.NewString("accounting@test.com, notifications@test.com"), VshnControlApiMetaStatus: odooclient.NewString("{\"conditions\":[{\"type\":\"ConditionFoo\",\"status\":\"False\",\"lastTransitionTime\":\"" + statusTime.Format(time.RFC3339) + "\",\"reason\":\"Whatever\",\"message\":\"Hello World\"}]}"), - }}), - mock.EXPECT().Read(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).SetArg(3, []odooclient.ResPartner{{ + }}, nil), + mock.EXPECT().FindResPartners(gomock.Any(), gomock.Any()).Return(&odooclient.ResPartners{{ Id: odooclient.NewInt(700), Name: odooclient.NewString("Test Company"), - }}), + }}, nil), ) s := &billingv1.BillingEntity{ diff --git a/apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/mock.go b/apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/odoo16.go similarity index 76% rename from apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/mock.go rename to apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/odoo16.go index 882bff8d..1b52fe22 100644 --- a/apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/mock.go +++ b/apiserver/billing/odoostorage/odoo/odoo16/odoo16mock/odoo16.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: apiserver/billing/odoostorage/odoo/odoo16/odoo16.go +// Source: github.com/appuio/control-api/apiserver/billing/odoostorage/odoo/odoo16 (interfaces: Odoo16Client) // // Generated by this command: // -// mockgen -source=apiserver/billing/odoostorage/odoo/odoo16/odoo16.go +// mockgen -destination=./odoo16mock/odoo16.go -package odoo16mock . Odoo16Client // // Package odoo16mock is a generated GoMock package. package odoo16mock @@ -11,7 +11,7 @@ package odoo16mock import ( reflect "reflect" - go_odoo "github.com/appuio/go-odoo" + odoo "github.com/appuio/go-odoo" gomock "go.uber.org/mock/gomock" ) @@ -39,7 +39,7 @@ func (m *MockOdoo16Client) EXPECT() *MockOdoo16ClientMockRecorder { } // CreateResPartner mocks base method. -func (m *MockOdoo16Client) CreateResPartner(arg0 *go_odoo.ResPartner) (int64, error) { +func (m *MockOdoo16Client) CreateResPartner(arg0 *odoo.ResPartner) (int64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateResPartner", arg0) ret0, _ := ret[0].(int64) @@ -68,10 +68,10 @@ func (mr *MockOdoo16ClientMockRecorder) DeleteResPartners(arg0 any) *gomock.Call } // FindResPartners mocks base method. -func (m *MockOdoo16Client) FindResPartners(arg0 *go_odoo.Criteria, arg1 *go_odoo.Options) (*go_odoo.ResPartners, error) { +func (m *MockOdoo16Client) FindResPartners(arg0 *odoo.Criteria, arg1 *odoo.Options) (*odoo.ResPartners, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FindResPartners", arg0, arg1) - ret0, _ := ret[0].(*go_odoo.ResPartners) + ret0, _ := ret[0].(*odoo.ResPartners) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -82,20 +82,6 @@ func (mr *MockOdoo16ClientMockRecorder) FindResPartners(arg0, arg1 any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindResPartners", reflect.TypeOf((*MockOdoo16Client)(nil).FindResPartners), arg0, arg1) } -// Read mocks base method. -func (m *MockOdoo16Client) Read(arg0 string, arg1 []int64, arg2 *go_odoo.Options, arg3 any) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Read", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// Read indicates an expected call of Read. -func (mr *MockOdoo16ClientMockRecorder) Read(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockOdoo16Client)(nil).Read), arg0, arg1, arg2, arg3) -} - // Update mocks base method. func (m *MockOdoo16Client) Update(arg0 string, arg1 []int64, arg2 any) error { m.ctrl.T.Helper() @@ -111,7 +97,7 @@ func (mr *MockOdoo16ClientMockRecorder) Update(arg0, arg1, arg2 any) *gomock.Cal } // UpdateResPartner mocks base method. -func (m *MockOdoo16Client) UpdateResPartner(arg0 *go_odoo.ResPartner) error { +func (m *MockOdoo16Client) UpdateResPartner(arg0 *odoo.ResPartner) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateResPartner", arg0) ret0, _ := ret[0].(error) diff --git a/apiserver/organization/billingentity_validate.go b/apiserver/organization/billingentity_validate.go index 4cedf7ce..c04ac6f6 100644 --- a/apiserver/organization/billingentity_validate.go +++ b/apiserver/organization/billingentity_validate.go @@ -3,6 +3,7 @@ package organization import ( "context" "fmt" + "os" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" @@ -45,29 +46,38 @@ func (c impersonatorFromRestconf) Impersonate(u user.Info) (client.Client, error // billingEntityValidator validates that the billing entity exists and the requesting user has access to it. // it does so by impersonating the user and trying to get the billing entity. func (s *organizationStorage) billingEntityValidator(ctx context.Context, org, oldOrg *orgv1.Organization) error { - // check if changed - if oldOrg != nil && oldOrg.Spec.BillingEntityRef == org.Spec.BillingEntityRef { - return nil - } - // check if we allow empty billing entities - if org.Spec.BillingEntityRef == "" && s.allowEmptyBillingEntity { - return nil - } + validate := func() error { + // check if changed + if oldOrg != nil && oldOrg.Spec.BillingEntityRef == org.Spec.BillingEntityRef { + return nil + } + // check if we allow empty billing entities + if org.Spec.BillingEntityRef == "" && s.allowEmptyBillingEntity { + return nil + } - user, ok := request.UserFrom(ctx) - if !ok { - return fmt.Errorf("no user in context") - } + user, ok := request.UserFrom(ctx) + if !ok { + return fmt.Errorf("no user in context") + } - var be billingv1.BillingEntity - c, err := s.impersonator.Impersonate(user) - if err != nil { - return fmt.Errorf("failed to impersonate user: %w", err) - } + var be billingv1.BillingEntity + c, err := s.impersonator.Impersonate(user) + if err != nil { + return fmt.Errorf("failed to impersonate user: %w", err) + } + + if err := c.Get(ctx, client.ObjectKey{Name: org.Spec.BillingEntityRef}, &be); err != nil { + return err + } - if err := c.Get(ctx, client.ObjectKey{Name: org.Spec.BillingEntityRef}, &be); err != nil { - return err + return nil } - return nil + err := validate() + if err != nil && s.skipBillingEntityValidation { + fmt.Fprintf(os.Stderr, "Warning: billing entity validation for %q (ref: %q) would have failed: %v\n", org.Name, org.Spec.BillingEntityRef, err) + return nil + } + return err } diff --git a/apiserver/organization/organization.go b/apiserver/organization/organization.go index 35d17634..c27bb604 100644 --- a/apiserver/organization/organization.go +++ b/apiserver/organization/organization.go @@ -28,7 +28,7 @@ import ( // +kubebuilder:rbac:groups="flowcontrol.apiserver.k8s.io",resources=prioritylevelconfigurations;flowschemas,verbs=get;list;watch // New returns a new storage provider for Organizations -func New(clusterRoles *[]string, usernamePrefix *string, allowEmptyBillingEntity *bool) restbuilder.ResourceHandlerProvider { +func New(clusterRoles *[]string, usernamePrefix *string, allowEmptyBillingEntity, skipBillingEntityValidation *bool) restbuilder.ResourceHandlerProvider { return func(s *runtime.Scheme, g genericregistry.RESTOptionsGetter) (rest.Storage, error) { masterConfig := loopback.GetLoopbackMasterClientConfig() @@ -54,9 +54,10 @@ func New(clusterRoles *[]string, usernamePrefix *string, allowEmptyBillingEntity members: kubeMemberProvider{ Client: c, }, - usernamePrefix: *usernamePrefix, - impersonator: impersonatorFromRestconf{masterConfig, client.Options{Scheme: c.Scheme()}}, - allowEmptyBillingEntity: *allowEmptyBillingEntity, + usernamePrefix: *usernamePrefix, + impersonator: impersonatorFromRestconf{masterConfig, client.Options{Scheme: c.Scheme()}}, + allowEmptyBillingEntity: *allowEmptyBillingEntity, + skipBillingEntityValidation: *skipBillingEntityValidation, } return authwrapper.NewAuthorizedStorage(stor, metav1.GroupVersionResource{ @@ -78,7 +79,8 @@ type organizationStorage struct { impersonator impersonator - allowEmptyBillingEntity bool + skipBillingEntityValidation bool + allowEmptyBillingEntity bool } func (s organizationStorage) New() runtime.Object { diff --git a/migration/export_mapping/main.go b/migration/export_mapping/main.go new file mode 100644 index 00000000..a46019c8 --- /dev/null +++ b/migration/export_mapping/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/csv" + "fmt" + "os" + "strings" + + odooclient "github.com/appuio/go-odoo" +) + +var notInflightFilter = odooclient.NewCriterion("vshn_control_api_inflight", "=", false) +var includeArchivedFilter = odooclient.NewCriterion("active", "in", []bool{true, false}) + +var fetchPartnerFieldOpts = odooclient.NewOptions().FetchFields( + "id", + "type", + "name", + "display_name", + // "country_id", + // "commercial_partner_id", + // "contact_address", + + // "child_ids", + // "user_ids", + + // "email", + // "phone", + // "street", + // "street2", + // "city", + // "zip", + // "country_id", + + // "parent_id", + // "vshn_control_api_meta_status", + "vshn_control_api_inflight", + + "x_odoo_8_ID", +) + +func main() { + session, err := odooclient.NewClient(&odooclient.ClientConfig{ + Database: "VSHNProd", + Admin: "odoo-automation@vshn.ch", + Password: os.Getenv("ODOO_PASSWORD"), + URL: "https://central.vshn.ch/", + }) + if err != nil { + panic(err) + } + + criteria := odooclient.NewCriteria().AddCriterion(includeArchivedFilter).AddCriterion(notInflightFilter) + accPartners, err := session.FindResPartners(criteria, fetchPartnerFieldOpts) + if err != nil { + panic(err) + } + + csvw := csv.NewWriter(os.Stdout) + + for _, p := range *accPartners { + csvw.Write([]string{ + fmt.Sprintf("%d", p.Id.Get()), + strings.TrimPrefix(p.XOdoo8ID.Get(), "__export__.res_partner_"), + p.Name.Get(), + p.DisplayName.Get(), + formatType(p.Type), + }) + } + csvw.Flush() + if err := csvw.Error(); err != nil { + panic(err) + } +} + +func formatType(s *odooclient.Selection) string { + v := s.Get() + if v == nil { + return "unknown" + } + return v.(string) +} diff --git a/migration/migrate/main.go b/migration/migrate/main.go new file mode 100644 index 00000000..9eba98a4 --- /dev/null +++ b/migration/migrate/main.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "encoding/csv" + "flag" + "fmt" + "io" + "os" + "regexp" + "slices" + "strings" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + billingv1 "github.com/appuio/control-api/apis/billing/v1" + orgv1 "github.com/appuio/control-api/apis/organization/v1" + userv1 "github.com/appuio/control-api/apis/user/v1" + controlv1 "github.com/appuio/control-api/apis/v1" +) + +var manualMapping = map[string]string{ + "2326": "9466", + "2746": "8019", + "2962": "16194", + "2667": "9143", + "1606": "8688", + "2778": "9366", + "2929": "16400", +} + +func main() { + ctx := context.Background() + + var dryRun, iCheckedInvitations, force, migrate bool + + flag.BoolVar(&dryRun, "dry-run", true, "dry run") + flag.BoolVar(&iCheckedInvitations, "i-checked-invitations", false, "i checked that there are no pending invitations for the billing entities") + flag.BoolVar(&force, "force", false, "override checks") + flag.BoolVar(&migrate, "migrate", false, "do migration") + + flag.Parse() + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(orgv1.AddToScheme(scheme)) + utilruntime.Must(controlv1.AddToScheme(scheme)) + utilruntime.Must(billingv1.AddToScheme(scheme)) + utilruntime.Must(userv1.AddToScheme(scheme)) + + c, err := client.New(ctrl.GetConfigOrDie(), client.Options{ + Scheme: scheme, + }) + if err != nil { + panic(err) + } + + old2new, _, meta, err := loadMapping() + if err != nil { + panic(err) + } + + for id, newID := range manualMapping { + old2new[id] = newID + } + + var es billingv1.BillingEntityList + if err := c.List(ctx, &es); err != nil { + panic(err) + } + + manifests, err := collectManifestsRequiringMigration(ctx, c) + if err != nil { + panic(err) + } + if _, ok := manifests[""]; ok { + fmt.Fprintln(os.Stderr, "Found manifests without billing entity") + os.Exit(1) + } + + var missing []string + type wrongType struct{ id, newId, t string } + var wrongTypes []wrongType + for id := range manifests { + newId, ok := old2new[id] + if !ok { + missing = append(missing, id) + continue + } + if t := meta[newId].Type; t != "invoice" { + wrongTypes = append(wrongTypes, wrongType{id, newId, t}) + } + } + slices.Sort(missing) + if len(missing) > 0 { + fmt.Fprintln(os.Stderr, "Missing mappings for", missing) + if !force { + os.Exit(1) + } + } + if len(wrongTypes) > 0 { + fmt.Fprintf(os.Stderr, "Wrong types for %+v", wrongTypes) + if !force { + os.Exit(1) + } + } + + if !iCheckedInvitations { + fmt.Fprintln(os.Stderr, "Make sure there are no pending invitations for the billing entities") + os.Exit(1) + } + + fmt.Fprintln(os.Stderr, "All checks passed") + + if !migrate { + return + } + + fmt.Fprintln(os.Stderr, "Deleting old RBAC") + deleteRBAC(ctx, c) + + fmt.Fprintln(os.Stderr, "Migrating manifests") + + for id, ms := range manifests { + if old2new[id] == "" && force { + fmt.Fprintln(os.Stderr, "Skipping", id) + continue + } + fmt.Fprintln(os.Stderr, "Migrating", id, "->", old2new[id]) + for _, m := range ms { + switch m := m.(type) { + case *rbacv1.ClusterRole: + fmt.Fprintln(os.Stderr, "Skipping role", m.Name, "will be recreated by the controller") + case *rbacv1.ClusterRoleBinding: + opts := []client.CreateOption{} + if dryRun { + opts = append(opts, client.DryRunAll) + } + pf := roleBeRegexp.FindStringSubmatch(m.Name) + crb := m.DeepCopy() + crb.ObjectMeta = metav1.ObjectMeta{ + Name: "billingentities-be-" + old2new[id] + "-" + pf[2], + Labels: map[string]string{ + "appuio.io/odoo-migrated": "true", + }, + } + fmt.Fprintln(os.Stderr, "Migrating role binding", m.Name, "->", crb.Name) + if err := c.Create(ctx, crb, opts...); err != nil { + panic(err) + } + case *orgv1.Organization: + m.Labels["appuio.io/odoo-migrated"] = "true" + fmt.Fprintln(os.Stderr, "Migrating org", m.Name, m.Spec.BillingEntityRef, "->", "be-"+old2new[id]) + m.Spec.BillingEntityRef = "be-" + old2new[id] + // we don't implement dry run correctly + if !dryRun { + if err := c.Update(ctx, m); err != nil { + panic(err) + } + } + } + } + } +} + +func deleteRBAC(ctx context.Context, c client.Client) { + var crbs rbacv1.ClusterRoleBindingList + if err := c.List(ctx, &crbs); err != nil { + panic(err) + } + var crs rbacv1.ClusterRoleList + if err := c.List(ctx, &crs); err != nil { + panic(err) + } + for _, crb := range crbs.Items { + if !strings.HasPrefix(crb.Name, "billingentities-be-") { + continue + } + fmt.Fprintln(os.Stderr, "Deleting binding", crb.Name) + if err := c.Delete(ctx, &crb, client.DryRunAll); err != nil { + panic(err) + } + } + for _, cr := range crs.Items { + if !strings.HasPrefix(cr.Name, "billingentities-be-") { + continue + } + fmt.Fprintln(os.Stderr, "Deleting role", cr.Name) + if err := c.Delete(ctx, &cr, client.DryRunAll); err != nil { + panic(err) + } + } +} + +type recordMeta struct { + Type string +} + +// loadMapping loads the mapping.csv file and compares the data with the data +func loadMapping() (old2new map[string]string, new2old map[string]string, meta map[string]recordMeta, err error) { + old2new = make(map[string]string) + new2old = make(map[string]string) + meta = make(map[string]recordMeta) + + cr := csv.NewReader(os.Stdin) + for { + record, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to read record: %w", err) + } + if record[1] == "" { + fmt.Fprintln(os.Stderr, "no old id for", record[0], record[2], record[3], "found") + continue + } + old2new[record[1]] = record[0] + new2old[record[0]] = record[1] + + meta[record[0]] = recordMeta{ + Type: record[4], + } + } + return old2new, new2old, meta, nil +} + +var roleBeRegexp = regexp.MustCompile(`^billingentities-be-(\d+)-(.+)$`) + +func collectManifestsRequiringMigration(ctx context.Context, c client.Client) (map[string][]client.Object, error) { + manifests := map[string][]client.Object{} + + findCr := func(crs rbacv1.ClusterRoleList, name string) (rbacv1.ClusterRole, bool) { + for _, cr := range crs.Items { + if cr.Name == name { + return cr, true + } + } + return rbacv1.ClusterRole{}, false + } + + var crbs rbacv1.ClusterRoleBindingList + if err := c.List(ctx, &crbs); err != nil { + return nil, fmt.Errorf("failed to list cluster role bindings: %w", err) + } + var crs rbacv1.ClusterRoleList + if err := c.List(ctx, &crs); err != nil { + return nil, fmt.Errorf("failed to list cluster roles: %w", err) + } + for _, crb := range crbs.Items { + crb := crb + if !strings.HasPrefix(crb.Name, "billingentities-be-") { + continue + } + if len(crb.Subjects) == 0 { + continue + } + m := roleBeRegexp.FindStringSubmatch(crb.Name) + if m == nil { + fmt.Fprintln(os.Stderr, "can't parse", crb.Name) + continue + } + id := m[1] + + manifests[id] = append(manifests[id], &crb) + cr, ok := findCr(crs, crb.Name) + if ok { + manifests[id] = append(manifests[id], &cr) + } + } + + var orgs orgv1.OrganizationList + if err := c.List(ctx, &orgs); err != nil { + return nil, fmt.Errorf("failed to list organizations: %w", err) + } + for _, org := range orgs.Items { + org := org + if org.Spec.BillingEntityRef == "" { + fmt.Fprintln(os.Stderr, "skipping", org.Name, "no billing entity ref") + continue + } + id := strings.TrimPrefix(org.Spec.BillingEntityRef, "be-") + manifests[id] = append(manifests[id], &org) + } + + return manifests, nil +} diff --git a/migration/playbook.md b/migration/playbook.md new file mode 100644 index 00000000..72293fed --- /dev/null +++ b/migration/playbook.md @@ -0,0 +1,21 @@ +# migration playbook + +- [ ] Run migration check until no failures are reported `ODOO_PASSWORD="..." go run ./migration/export_mapping | go run ./migration/migrate` + - Fill manual mappings +- [ ] Dump mapping state `go run ./migration/export_mapping > mapping.csv` +- [ ] Maintenance announcement https://statuspal.eu/admin/status_pages/appuio-cloud (https://vs.hn/statuspal) +- [ ] Switch off portal.appuio.cloud + - Disable lpg2 ArgoCD sync for root and appuio-portal, delete the ingress +- [ ] Disable control-api controller `k -n appuio-control-api scale deployment control-api-controller --replicas=0` +- [ ] Disable control-api controller `--sale-order-compatibility-mode` in tenant repo +- [ ] Switch control-api apiserver to fake billing storage +- [ ] Check for pending invitations, delete if necessary (if they contain billing entities) +- [ ] Rerun migration checks `go run ./migration/migrate -i-checked-invitations < mapping.csv` +- [ ] Execute migration dry run `go run ./migration/migrate -i-checked-invitations -migrate < mapping.csv` +- [ ] Execute migration `go run ./migration/migrate -i-checked-invitations -migrate -dry-run=false < mapping.csv` +- [ ] Check if any manifest left without migration label + - `k get organization -l appuio.io/odoo-migrated!=true` + - `k get clusterroles -l appuio.io/odoo-migrated!=true | grep billingentities-` + - `k get clusterrolebindings -l appuio.io/odoo-migrated!=true | grep billingentities-` +- [ ] Resync app +- [ ] Resync lpg2 root