feat: read files from INITIAL_DATA_DIRECTORY

Author: adamdecaf

README.md

@@ -188,18 +188,21 @@ PONG
 
 ### Configuration settings
 
-| Environmental Variable | Description                                                                                                                         | Default |
-|-----|-------------------------------------------------------------------------------------------------------------------------------------|-----|
-| `FEDACH_DATA_PATH` | Filepath to FedACH data file                                                                                                        | `./data/FedACHdir.txt` |
-| `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file                                                                                                       | `./data/fpddir.txt` |
-| `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files                                      | Empty |
-| `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files                                       | Empty |
-| `FRB_DOWNLOAD_URL_TEMPLATE` | URL Template for downloading files from alternate source                                                                            | `https://frbservices.org/EPaymentsDirectory/directories/%s?format=json`|
-| `LOG_FORMAT` | Format for logging lines to be written as.                                                                                          | Options: `json`, `plain` - Default: `plain` |
-| `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`.                                      | Default: `:8086` |
-| `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`.                               | Default: `:9096` |
-| `HTTPS_CERT_FILE` | Filepath containing a certificate (or intermediate chain) to be served by the HTTP server. Requires all traffic be over secure HTTP. | Empty |
-| `HTTPS_KEY_FILE`  | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`.                                                     | Empty |
+| Environmental Variable      | Description                                                                                                                          | Default                                                                 |
+|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
+| `FEDACH_DATA_PATH`          | Filepath to FedACH data file                                                                                                         | `./data/FedACHdir.txt`                                                  |
+| `FEDWIRE_DATA_PATH`         | Filepath to Fedwire data file                                                                                                        | `./data/fpddir.txt`                                                     |
+| `INITIAL_DATA_DIRECTORY`    | Directory of files to be used instead of downloading or `*_DATA_PATH` variables.                                                     | Empty                                                                   |
+|                        | ACH: FedACHdir.txt, fedachdir.json, fedach.txt, fedach.json                                                                          |                                                                    |
+|                        | Wire: fpddir.json, fpddir.txt, fedwire.txt, fedwire.json                                                                             |                                                                    |
+| `FRB_ROUTING_NUMBER`        | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files                                       | Empty                                                                   |
+| `FRB_DOWNLOAD_CODE`         | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files                                        | Empty                                                                   |
+| `FRB_DOWNLOAD_URL_TEMPLATE` | URL Template for downloading files from alternate source                                                                             | `https://frbservices.org/EPaymentsDirectory/directories/%s?format=json` |
+| `LOG_FORMAT`                | Format for logging lines to be written as.                                                                                           | Options: `json`, `plain` - Default: `plain`                             |
+| `HTTP_BIND_ADDRESS`         | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`.                                       | Default: `:8086`                                                        |
+| `HTTP_ADMIN_BIND_ADDRESS`   | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`.                                | Default: `:9096`                                                        |
+| `HTTPS_CERT_FILE`           | Filepath containing a certificate (or intermediate chain) to be served by the HTTP server. Requires all traffic be over secure HTTP. | Empty                                                                   |
+| `HTTPS_KEY_FILE`            | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`.                                                      | Empty                                                                   |
 
 ### Data persistence
 By design, Fed  **does not persist** (save) any data about the search queries created. The only storage occurs in memory of the process and upon restart Fed will have no files or data saved. Also, no in-memory encryption of the data is performed.

cmd/server/reader.go

@@ -9,21 +9,37 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/fed"
 	"github.com/moov-io/fed/pkg/download"
 )
 
+var (
+	fedachFilenames  = []string{"FedACHdir.txt", "fedachdir.json", "fedach.txt", "fedach.json"}
+	fedwireFilenames = []string{"fpddir.json", "fpddir.txt", "fedwire.txt", "fedwire.json"}
+)
+
 func fedACHDataFile(logger log.Logger) (io.Reader, error) {
-	file, err := attemptFileDownload(logger, "fedach")
+	initialDir := os.Getenv("INITIAL_DATA_DIRECTORY")
+	file, err := inspectInitialDataDirectory(logger, initialDir, fedachFilenames)
+	if err != nil {
+		return nil, fmt.Errorf("inspecting %s for FedACH file failed: %w", initialDir, err)
+	}
+	if file != nil {
+		logger.Info().Logf("found FedACH file in %s", initialDir)
+		return file, nil
+	}
+
+	file, err = attemptFileDownload(logger, "fedach")
 	if err != nil && !errors.Is(err, download.ErrMissingConfigValue) {
 		return nil, fmt.Errorf("problem downloading fedach: %v", err)
 	}
 
 	if file != nil {
 		logger.Info().Log("search: downloaded ACH file")
-
 		return file, nil
 	}
 
@@ -38,14 +54,23 @@ func fedACHDataFile(logger log.Logger) (io.Reader, error) {
 }
 
 func fedWireDataFile(logger log.Logger) (io.Reader, error) {
-	file, err := attemptFileDownload(logger, "fedwire")
+	initialDir := os.Getenv("INITIAL_DATA_DIRECTORY")
+	file, err := inspectInitialDataDirectory(logger, initialDir, fedwireFilenames)
+	if err != nil {
+		return nil, fmt.Errorf("inspecting %s for FedWire file failed: %w", initialDir, err)
+	}
+	if file != nil {
+		logger.Info().Logf("found FedWire file in %s", initialDir)
+		return file, nil
+	}
+
+	file, err = attemptFileDownload(logger, "fedwire")
 	if err != nil && !errors.Is(err, download.ErrMissingConfigValue) {
 		return nil, fmt.Errorf("problem downloading fedwire: %v", err)
 	}
 
 	if file != nil {
 		logger.Info().Log("search: downloaded Wire file")
-
 		return file, nil
 	}
 
@@ -59,6 +84,34 @@ func fedWireDataFile(logger log.Logger) (io.Reader, error) {
 	return file, nil
 }
 
+func inspectInitialDataDirectory(logger log.Logger, dir string, needles []string) (io.Reader, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("readdir on %s failed: %w", dir, err)
+	}
+
+	for _, entry := range entries {
+		_, filename := filepath.Split(entry.Name())
+
+		for idx := range needles {
+			if strings.EqualFold(filename, needles[idx]) {
+				where := filepath.Join(dir, entry.Name())
+
+				fd, err := os.Open(where)
+				if err != nil {
+					return nil, fmt.Errorf("opening %s failed: %w", where, err)
+				}
+				return fd, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
 func attemptFileDownload(logger log.Logger, listName string) (io.Reader, error) {
 	logger.Logf("download: attempting %s", listName)
 	client, err := download.NewClient(nil)

cmd/server/reader_test.go

@@ -32,6 +32,33 @@ func TestReader__fedWireDataFile(t *testing.T) {
 	require.ErrorContains(t, err, "no such file or directory")
 }
 
+func TestReader_inspectInitialDataDirectory(t *testing.T) {
+	logger := log.NewNopLogger()
+
+	dir := t.TempDir()
+
+	err := os.WriteFile(filepath.Join(dir, "fedach.txt"), nil, 0600)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(dir, "fedwire.txt"), nil, 0600)
+	require.NoError(t, err)
+
+	// FedACH files
+	fd, err := inspectInitialDataDirectory(logger, dir, fedachFilenames)
+	require.NoError(t, err)
+
+	file, ok := fd.(*os.File)
+	require.True(t, ok)
+	require.Equal(t, filepath.Join(dir, "fedach.txt"), file.Name())
+
+	// FedWire files
+	fd, err = inspectInitialDataDirectory(logger, dir, fedwireFilenames)
+	require.NoError(t, err)
+
+	file, ok = fd.(*os.File)
+	require.True(t, ok)
+	require.Equal(t, filepath.Join(dir, "fedwire.txt"), file.Name())
+}
+
 func TestReader__readFEDACHData(t *testing.T) {
 	s := &searcher{logger: log.NewNopLogger()}