implement import mode for the etcd datasource

README.md

@@ -12,6 +12,7 @@ A CLI tool (and a library) that processes sets of host attributes stored as DNS
 - **(DNS data source)** two modes of operation: zone transfers and regular DNS queries.
 - **(DNS data source)** TSIG support for zone transfers.
 - **(Etcd data source)** authentication and mTLS support.
+- **(Etcd data source)** importing host records from a YAML file.
 - Unlimited number and length of inventory tree branches.
 - Predictable and stable inventory structure.
 - Multiple records per host supported.
@@ -23,21 +24,23 @@ A CLI tool (and a library) that processes sets of host attributes stored as DNS
 ```txt
 Usage of dns-inventory:
   -attrs
-        export host attributes
+    	export host attributes
   -format string
-        select export format, if available (default "yaml")
+    	select export format, if available (default "yaml")
   -groups
-        export groups
+    	export groups
   -host string
-        produce a JSON dictionary of host variables for Ansible
+    	produce a JSON dictionary of host variables for Ansible
   -hosts
-        export hosts
+    	export hosts
+  -import string
+    	import host records from file
   -list
-        produce a JSON inventory for Ansible
+    	produce a JSON inventory for Ansible
   -tree
-        export raw inventory tree
+    	export raw inventory tree
   -version
-        display ansible-dns-inventory version and build info
+    	display ansible-dns-inventory version and build info
 ```
 
 ## Prerequisites
@@ -142,7 +145,7 @@ You create a key/value pair where the value is formatted the same way as with th
 
 
 
-### Host attributes (default key names)
+### Host attributes (default keys)
 
 | Key  | Description                                                                                                                                                 |
 | ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -152,8 +155,8 @@ You create a key/value pair where the value is formatted the same way as with th
 | SRV  | Host service identifier(s). This will be split further using the `txt.keys.separator` to produce a hierarchy of groups. Required. Can also be a comma-delimited list. |
 | VARS | Optional host variables.                                                                                                                                    |
 
-All key names and separators are customizable via `ansible-dns-inventory`'s config file.
-Key values are validated and can only contain numbers and letters of the Latin alphabet, except for the service identifier(s) which can also contain the `txt.keys.separator` symbol.
+All keys and separators are customizable via `ansible-dns-inventory`'s config file.
+Values are validated and can only contain numbers and letters of the Latin alphabet, except for the service identifier(s) which can also contain the `txt.keys.separator` symbol.
 
 ### Host variables
 
@@ -262,9 +265,38 @@ $ dns-inventory -attrs -format yaml-flow
 ...
 ```
 
+## Import mode
+
+Some `ansible-dns-inventory` datasources support importing host records from a YAML file. These currently include:
+- etcd datasource
+
+To populate one of these datasources with host records, first create a YAML file with the same structure as the `-attrs` export mode output:
+```
+# cat import.yaml
+app01.infra.local:
+- ENV: dev
+  OS: linux
+  ROLE: app
+  SRV: tomcat_backend_auth
+  VARS: ansible_host=10.0.0.1
+app02.infra.local:
+- ENV: dev
+  OS: linux
+  ROLE: app
+  SRV: tomcat_backend_auth
+  VARS: ansible_host=10.0.0.2
+```   
+
+Then run `ansible-dns-inventory` in the import mode:
+```
+dns-inventory -import ./import.yaml
+```
+
+WARNING: while only default host attribute keys (`OS/ENV/ROLE/SRV/VARS`) are supported in the input file itself, the actual records will use your custom keys if set in the configuration.
+
 ## Roadmap
 
 - [x] Implement key-value stores support (etcd, Consul, etc.).
 - [x] Support using `ansible-dns-inventory` as a library.
+- [!] Implement import mode for some of the datasources. (implemented for the etcd datasource)
 - [ ] Support more datasource types.
-- [ ] Add editor mode for some of the datasources.

cmd/dns-inventory/dns-inventory.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/NeonSludge/ansible-dns-inventory/internal/build"
 	"github.com/NeonSludge/ansible-dns-inventory/internal/config"
 	"github.com/NeonSludge/ansible-dns-inventory/internal/logger"
@@ -20,7 +22,8 @@ func main() {
 	groupsFlag := flag.Bool("groups", false, "export groups")
 	treeFlag := flag.Bool("tree", false, "export raw inventory tree")
 	formatFlag := flag.String("format", "yaml", "select export format, if available")
-	hostFlag := flag.String("host", "", "a stub for Ansible")
+	hostFlag := flag.String("host", "", "produce a JSON dictionary of host variables for Ansible")
+	importFlag := flag.String("import", "", "import host records from file")
 	versionFlag := flag.Bool("version", false, "display ansible-dns-inventory version and build info")
 	flag.Parse()
 
@@ -47,7 +50,25 @@ func main() {
 	}
 	defer dnsInventory.Datasource.Close()
 
-	if len(*hostFlag) == 0 {
+	if len(*importFlag) > 0 {
+		hosts := make(map[string][]*inventory.HostAttributes)
+
+		importFile, err := os.ReadFile(*importFlag)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = yaml.Unmarshal(importFile, hosts)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Infof("importing hosts from file: %s", *importFlag)
+
+		if err := dnsInventory.PublishHosts(hosts); err != nil {
+			log.Fatal(err)
+		}
+	} else if len(*hostFlag) == 0 {
 		var bytes []byte
 		var err error
 

pkg/inventory/dns.go

@@ -73,6 +73,26 @@ func (d *DNSDatasource) makeFQDN(host string, zone string) string {
 	return strings.TrimPrefix(dns.Fqdn(name+"."+domain), ".")
 }
 
+// findZone selects a matching zone from the datasource configuration based on the hostname.
+func (d *DNSDatasource) findZone(host string) (string, error) {
+	cfg := d.Config
+	var zone string
+
+	// Try finding a matching zone in the configuration.
+	for _, z := range cfg.DNS.Zones {
+		if strings.HasSuffix(strings.Trim(host, "."), strings.Trim(z, ".")) {
+			zone = z
+			break
+		}
+	}
+
+	if len(zone) == 0 {
+		return zone, errors.New("no matching zones found in config file")
+	}
+
+	return zone, nil
+}
+
 // getZone acquires TXT records for all hosts in a specific zone.
 func (d *DNSDatasource) getZone(zone string) ([]dns.RR, error) {
 	cfg := d.Config
@@ -148,23 +168,14 @@ func (d *DNSDatasource) GetAllRecords() ([]*DatasourceRecord, error) {
 func (d *DNSDatasource) GetHostRecords(host string) ([]*DatasourceRecord, error) {
 	cfg := d.Config
 	records := make([]*DatasourceRecord, 0)
-	var err error
 
 	if cfg.DNS.Notransfer.Enabled {
 		// No-transfer mode is enabled.
-		var zone string
 		var rrs []dns.RR
 
-		// Determine which zone we are working with.
-		for _, z := range cfg.DNS.Zones {
-			if strings.HasSuffix(strings.Trim(host, "."), strings.Trim(z, ".")) {
-				zone = z
-				break
-			}
-		}
-
-		if len(zone) == 0 {
-			return nil, errors.New("failed to determine zone from hostname")
+		zone, err := d.findZone(host)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to determine zone from hostname")
 		}
 
 		// Get no-transfer host records.
@@ -193,6 +204,14 @@ func (d *DNSDatasource) GetHostRecords(host string) ([]*DatasourceRecord, error)
 	return records, nil
 }
 
+// PublishRecords writes host records to the datasource.
+func (d *DNSDatasource) PublishRecords(records []*DatasourceRecord) error {
+	log := d.Logger
+
+	log.Warn("Publishing records has not been implemented for the DNS datasource yet.")
+	return nil
+}
+
 // Close shuts down the datasource and performs other housekeeping.
 func (d *DNSDatasource) Close() {}
 

pkg/inventory/etcd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -68,6 +69,26 @@ func (e *EtcdDatasource) processKVs(kvs []*mvccpb.KeyValue) []*DatasourceRecord
 	return records
 }
 
+// findZone selects a matching zone from the datasource configuration based on the hostname.
+func (e *EtcdDatasource) findZone(host string) (string, error) {
+	cfg := e.Config
+	var zone string
+
+	// Try finding a matching zone in the configuration.
+	for _, z := range cfg.Etcd.Zones {
+		if strings.HasSuffix(strings.Trim(host, "."), strings.Trim(z, ".")) {
+			zone = z
+			break
+		}
+	}
+
+	if len(zone) == 0 {
+		return zone, errors.New("no matching zones found in config file")
+	}
+
+	return zone, nil
+}
+
 // getPrefix acquires all key/value records for a specific prefix.
 func (e *EtcdDatasource) getPrefix(prefix string) ([]*mvccpb.KeyValue, error) {
 	cfg := e.Config
@@ -81,6 +102,25 @@ func (e *EtcdDatasource) getPrefix(prefix string) ([]*mvccpb.KeyValue, error) {
 	return resp.Kvs, nil
 }
 
+// putRecord publishes a host record via the datasource.
+func (e *EtcdDatasource) putRecord(record *DatasourceRecord, count int) error {
+	cfg := e.Config
+
+	zone, err := e.findZone(record.Hostname)
+	if err != nil {
+		return errors.Wrap(err, "failed to determine zone from hostname")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), cfg.Etcd.Timeout)
+	_, err = e.Client.Put(ctx, fmt.Sprintf("%s/%s/%d", zone, record.Hostname, count), record.Attributes)
+	cancel()
+	if err != nil {
+		return errors.Wrap(err, "etcd request failure")
+	}
+
+	return nil
+}
+
 // GetAllRecords acquires all available host records.
 func (e *EtcdDatasource) GetAllRecords() ([]*DatasourceRecord, error) {
 	cfg := e.Config
@@ -102,19 +142,9 @@ func (e *EtcdDatasource) GetAllRecords() ([]*DatasourceRecord, error) {
 
 // GetHostRecords acquires all available records for a specific host.
 func (e *EtcdDatasource) GetHostRecords(host string) ([]*DatasourceRecord, error) {
-	cfg := e.Config
-	var zone string
-
-	// Determine which zone we are working with.
-	for _, z := range cfg.Etcd.Zones {
-		if strings.HasSuffix(strings.Trim(host, "."), strings.Trim(z, ".")) {
-			zone = z
-			break
-		}
-	}
-
-	if len(zone) == 0 {
-		return nil, errors.New("failed to determine zone from hostname")
+	zone, err := e.findZone(host)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine zone from hostname")
 	}
 
 	prefix := zone + "/" + host
@@ -126,6 +156,25 @@ func (e *EtcdDatasource) GetHostRecords(host string) ([]*DatasourceRecord, error
 	return e.processKVs(kvs), nil
 }
 
+// PublishRecords writes host records to the datasource.
+func (e *EtcdDatasource) PublishRecords(records []*DatasourceRecord) error {
+	counts := map[string]int{}
+
+	for _, record := range records {
+		if _, ok := counts[record.Hostname]; ok {
+			counts[record.Hostname]++
+		} else {
+			counts[record.Hostname] = 0
+		}
+
+		if err := e.putRecord(record, counts[record.Hostname]); err != nil {
+			return errors.Wrap(err, "failed to publish a host record")
+		}
+	}
+
+	return nil
+}
+
 // Close shuts down the datasource and performs other housekeeping.
 func (e *EtcdDatasource) Close() {
 	e.Client.Close()

pkg/inventory/inventory.go

@@ -175,6 +175,54 @@ func (i *Inventory) ParseAttributes(raw string) (*HostAttributes, error) {
 	return attrs, nil
 }
 
+// RenderAttributes constructs a string representation of the HostAttributes struct.
+func (i *Inventory) RenderAttributes(attributes *HostAttributes) (string, error) {
+	cfg := i.Config
+
+	attrString := strings.Builder{}
+
+	if err := i.Validator.Struct(attributes); err != nil {
+		return "", errors.Wrap(err, "attribute validation error")
+	}
+
+	attrs := [][]string{{cfg.Txt.Keys.Os, attributes.OS}, {cfg.Txt.Keys.Env, attributes.Env}, {cfg.Txt.Keys.Role, attributes.Role}, {cfg.Txt.Keys.Srv, attributes.Srv}, {cfg.Txt.Keys.Vars, attributes.Vars}}
+
+	for i, attr := range attrs {
+		attrString.WriteString(attr[0])
+		attrString.WriteString(cfg.Txt.Kv.Equalsign)
+		attrString.WriteString(attr[1])
+
+		if i != len(attrs)-1 {
+			attrString.WriteString(cfg.Txt.Kv.Separator)
+		}
+	}
+
+	return attrString.String(), nil
+}
+
+// PublishHosts publishes host records via the datasource.
+func (i *Inventory) PublishHosts(hosts map[string][]*HostAttributes) error {
+	log := i.Logger
+
+	records := []*DatasourceRecord{}
+
+	for hostname, attrsList := range hosts {
+		for _, attrs := range attrsList {
+			if attrString, err := i.RenderAttributes(attrs); err == nil {
+				records = append(records, &DatasourceRecord{
+					Hostname:   hostname,
+					Attributes: attrString,
+				})
+			} else {
+				log.Warnf("[%s] skipping host record: %v", hostname, err)
+				continue
+			}
+		}
+	}
+
+	return i.Datasource.PublishRecords(records)
+}
+
 // New creates an instance of the DNS inventory with user-supplied configuration.
 func New(cfg *Config) (*Inventory, error) {
 	// Setup package global state