(Feat): Initial Commit

internal/api/handlers/common/get_healthy.go

@@ -0,0 +1,55 @@
+// nolint:revive
+package common
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"github.com/labstack/echo/v4"
+)
+
+const (
+	// We use 521 to indicate an error state
+	// same as Cloudflare: https://support.cloudflare.com/hc/en-us/articles/115003011431#521error
+	httpStatusDown = 521
+)
+
+func GetHealthyRoute(s *api.Server) *echo.Route {
+	return s.Router.Management.GET("/healthy", getHealthyHandler(s))
+}
+
+// Heathly check (= liveness)
+// Returns an human readable string about the current service status.
+// In addition to readiness probes, it performs actual write probes.
+// Note that /-/healthy is private (shielded by the mgmt-secret) as it may expose sensitive information about your service.
+// Structured upon https://prometheus.io/docs/prometheus/latest/management_api/
+func getHealthyHandler(s *api.Server) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		if !s.Ready() {
+			return c.String(httpStatusDown, "Not ready.")
+		}
+
+		var str strings.Builder
+		fmt.Fprintln(&str, "Ready.")
+
+		// General Timeout and associated context.
+		ctx, cancel := context.WithTimeout(c.Request().Context(), s.Config.Management.LivenessTimeout)
+		defer cancel()
+
+		healthyStr, errs := ProbeLiveness(ctx, s.DB, s.Config.Management.ProbeWriteablePathsAbs, s.Config.Management.ProbeWriteableTouchfile)
+		str.WriteString(healthyStr)
+
+		// Finally return the health status according to the seen states
+		if ctx.Err() != nil || len(errs) != 0 {
+			fmt.Fprintln(&str, "Probes failed.")
+			return c.String(httpStatusDown, str.String())
+		}
+
+		fmt.Fprintln(&str, "Probes succeeded.")
+
+		return c.String(http.StatusOK, str.String())
+	}
+}

internal/api/handlers/common/get_healthy_test.go

@@ -0,0 +1,110 @@
+package common_test
+
+import (
+	"net/http"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHealthySuccess(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// explicitly set touchfile that no other test has (so we can explicitly remove it beforehand.)
+		s.Config.Management.ProbeWriteableTouchfile = ".healthy-test"
+
+		for _, writeablePath := range s.Config.Management.ProbeWriteablePathsAbs {
+			os.Remove(path.Join(writeablePath, s.Config.Management.ProbeWriteableTouchfile))
+
+			// also remove after test completion.
+			defer os.Remove(path.Join(writeablePath, s.Config.Management.ProbeWriteableTouchfile))
+		}
+
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, http.StatusOK, res.Result().StatusCode)
+		require.Contains(t, res.Body.String(), "seq_health=1")
+
+		firstTouchTime := make([]time.Time, len(s.Config.Management.ProbeWriteablePathsAbs))
+
+		// expect a new touchfiles were written
+		for i, writeablePath := range s.Config.Management.ProbeWriteablePathsAbs {
+			filePath := path.Join(writeablePath, s.Config.Management.ProbeWriteableTouchfile)
+			stat, err := os.Stat(filePath)
+			require.NoErrorf(t, err, "Expected to have %v", filePath)
+			firstTouchTime[i] = stat.ModTime()
+		}
+
+		res = test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, http.StatusOK, res.Result().StatusCode)
+		require.Contains(t, res.Body.String(), "seq_health=2")
+
+		// expect touchfiles modTime was updated
+		for i, writeablePath := range s.Config.Management.ProbeWriteablePathsAbs {
+			filePath := path.Join(writeablePath, s.Config.Management.ProbeWriteableTouchfile)
+			stat, err := os.Stat(filePath)
+			require.NoErrorf(t, err, "Expected to have %v", filePath)
+
+			assert.NotEqual(t, firstTouchTime[i], stat.ModTime())
+		}
+	})
+}
+
+func TestGetHealthyNoAuth(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		res := test.PerformRequest(t, s, "GET", "/-/healthy", nil, nil)
+		require.Equal(t, http.StatusBadRequest, res.Result().StatusCode)
+	})
+}
+
+func TestGetHealthyWrongAuth(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret=i-have-no-idea-about-the-pass", nil, nil)
+		require.Equal(t, http.StatusUnauthorized, res.Result().StatusCode)
+	})
+}
+
+func TestGetHealthyDBPingError(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// forcefully close the DB
+		s.DB.Close()
+
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+	})
+}
+
+func TestGetHealthyDBSeqError(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// forcefully remove the sequence
+		if _, err := s.DB.Exec("DROP SEQUENCE seq_health;"); err != nil {
+			t.Fatal(err, "was unable to drop sequence seq_health")
+		}
+
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+	})
+}
+
+func TestGetHealthyMountError(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		s.Config.Management.ProbeWriteablePathsAbs = []string{"/this/path/does/not/exist"}
+
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+	})
+}
+
+func TestGetHealthyNotReady(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// forcefully remove an initialized component to check if ready state works
+		s.Mailer = nil
+
+		res := test.PerformRequest(t, s, "GET", "/-/healthy?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+	})
+}

internal/api/handlers/common/get_ready.go

@@ -0,0 +1,40 @@
+// nolint:revive
+package common
+
+import (
+	"context"
+	"net/http"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"github.com/labstack/echo/v4"
+)
+
+func GetReadyRoute(s *api.Server) *echo.Route {
+	return s.Router.Management.GET("/ready", getReadyHandler(s))
+}
+
+// Readiness check
+// This endpoint returns 200 when our Service is ready to serve traffic (i.e. respond to queries).
+// Does read-only probes apart from the general server ready state.
+// Note that /-/ready is typically public (and not shielded by a mgmt-secret), we thus prevent information leakage here and only return `"Ready."`.
+// Structured upon https://prometheus.io/docs/prometheus/latest/management_api/
+func getReadyHandler(s *api.Server) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		if !s.Ready() {
+			return c.String(httpStatusDown, "Not ready.")
+		}
+
+		// General Timeout and associated context.
+		ctx, cancel := context.WithTimeout(c.Request().Context(), s.Config.Management.ReadinessTimeout)
+		defer cancel()
+
+		_, errs := ProbeReadiness(ctx, s.DB, s.Config.Management.ProbeWriteablePathsAbs)
+
+		// Finally return the health status according to the seen states
+		if ctx.Err() != nil || len(errs) != 0 {
+			return c.String(httpStatusDown, "Not ready.")
+		}
+
+		return c.String(http.StatusOK, "Ready.")
+	}
+}

internal/api/handlers/common/get_ready_test.go

@@ -0,0 +1,41 @@
+package common_test
+
+import (
+	"net/http"
+	"testing"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetReadyReadiness(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		res := test.PerformRequest(t, s, "GET", "/-/ready", nil, nil)
+		require.Equal(t, http.StatusOK, res.Result().StatusCode)
+		require.Equal(t, "Ready.", res.Body.String())
+	})
+}
+
+func TestGetReadyReadinessBroken(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// forcefully remove an initialized component to check if ready state works
+		s.Mailer = nil
+
+		res := test.PerformRequest(t, s, "GET", "/-/ready", nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+		require.Equal(t, "Not ready.", res.Body.String())
+	})
+}
+
+func TestGetReadyDBBrokenNotReady(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		// forcefully remove pg
+		err := s.DB.Close()
+		require.NoError(t, err)
+
+		res := test.PerformRequest(t, s, "GET", "/-/ready", nil, nil)
+		require.Equal(t, 521, res.Result().StatusCode)
+		require.Equal(t, "Not ready.", res.Body.String())
+	})
+}

internal/api/handlers/common/get_swagger.go

@@ -0,0 +1,20 @@
+// nolint:revive
+package common
+
+import (
+	"path/filepath"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/api/middleware"
+	"github.com/labstack/echo/v4"
+)
+
+func GetSwaggerRoute(s *api.Server) *echo.Route {
+	// ---
+	// Serve generated swagger.yml file statically at /swagger.yml
+	// hack: not attached to group - can go away after echo/group.go .File and .Static actually return the *echo.Route
+	// see https://github.com/labstack/echo/issues/1595
+	// return s.Router.Root.File("swagger.yml", filepath.Join(s.Config.Echo.APIBaseDirAbs, "swagger.yml"))
+	// we explicitly enforce a no-cache directive on any requests to it.
+	return s.Echo.File("/swagger.yml", filepath.Join(s.Config.Paths.APIBaseDirAbs, "swagger.yml"), middleware.NoCache())
+}

internal/api/handlers/common/get_swagger_test.go

@@ -0,0 +1,32 @@
+package common_test
+
+import (
+	"net/http"
+	"testing"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSwaggerYAMLRetrieval(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		res := test.PerformRequest(t, s, "GET", "/swagger.yml", nil, nil)
+		require.Equal(t, http.StatusOK, res.Result().StatusCode)
+
+		// caching: ensure this call is always uncached for browsers
+		assert.Equal(t, "no-cache, private, max-age=0", res.Header().Get("Cache-Control"))
+		assert.Equal(t, "Thu, 01 Jan 1970 00:00:00 UTC", res.Header().Get("Expires"))
+		assert.Equal(t, "0", res.Header().Get("X-Accel-Expires"))
+		assert.Equal(t, "no-cache", res.Header().Get("Pragma"))
+
+		// caching: unset
+		assert.Empty(t, res.Header().Get("ETag"))
+		assert.Empty(t, res.Header().Get("If-Modified-Since"))
+		assert.Empty(t, res.Header().Get("If-Match"))
+		assert.Empty(t, res.Header().Get("If-None-Match"))
+		assert.Empty(t, res.Header().Get("If-Range"))
+		assert.Empty(t, res.Header().Get("If-Unmodified-Since"))
+	})
+}

internal/api/handlers/common/get_version.go

@@ -0,0 +1,21 @@
+// nolint:revive
+package common
+
+import (
+	"net/http"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/config"
+	"github.com/labstack/echo/v4"
+)
+
+func GetVersionRoute(s *api.Server) *echo.Route {
+	return s.Router.Management.GET("/version", getVersionHandler(s))
+}
+
+// Returns the version and build date baked into the binary.
+func getVersionHandler(_ *api.Server) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		return c.String(http.StatusOK, config.GetFormattedBuildArgs())
+	}
+}

internal/api/handlers/common/get_version_test.go

@@ -0,0 +1,33 @@
+package common_test
+
+import (
+	"net/http"
+	"testing"
+
+	"allaboutapps.dev/aw/go-starter/internal/api"
+	"allaboutapps.dev/aw/go-starter/internal/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetVersion(t *testing.T) {
+	test.WithTestServer(t, func(s *api.Server) {
+		res := test.PerformRequest(t, s, "GET", "/-/version?mgmt-secret="+s.Config.Management.Secret, nil, nil)
+		require.Equal(t, http.StatusOK, res.Result().StatusCode)
+		require.Equal(t, "build.local/misses/ldflags @ < 40 chars git commit hash via ldflags > (1970-01-01T00:00:00+00:00)", res.Body.String()) // build args are not injected during test time.
+
+		// caching: ensure this call is always uncached for browsers
+		assert.Equal(t, "no-cache, private, max-age=0", res.Header().Get("Cache-Control"))
+		assert.Equal(t, "Thu, 01 Jan 1970 00:00:00 UTC", res.Header().Get("Expires"))
+		assert.Equal(t, "0", res.Header().Get("X-Accel-Expires"))
+		assert.Equal(t, "no-cache", res.Header().Get("Pragma"))
+
+		// caching: unset
+		assert.Empty(t, res.Header().Get("ETag"))
+		assert.Empty(t, res.Header().Get("If-Modified-Since"))
+		assert.Empty(t, res.Header().Get("If-Match"))
+		assert.Empty(t, res.Header().Get("If-None-Match"))
+		assert.Empty(t, res.Header().Get("If-Range"))
+		assert.Empty(t, res.Header().Get("If-Unmodified-Since"))
+	})
+}

internal/api/handlers/common/probes.go

@@ -0,0 +1,226 @@
+// nolint:revive
+package common
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"allaboutapps.dev/aw/go-starter/internal/util"
+	"golang.org/x/sys/unix"
+)
+
+func ProbeReadiness(ctx context.Context, database *sql.DB, writeablePaths []string) (string, []error) {
+	var str strings.Builder
+
+	// slice collects all errors from probes
+	errs := make([]error, 0, 1+len(writeablePaths))
+
+	// DB readable?
+	dbPingStr, dbPingErr := probeDatabasePingable(ctx, database)
+	str.WriteString(dbPingStr)
+
+	if dbPingErr != nil {
+		errs = append(errs, dbPingErr)
+	}
+
+	// FS (potentially) writeable?
+	for _, writeablePath := range writeablePaths {
+		fsPermStr, fsPermErr := probePathWriteablePermission(ctx, writeablePath)
+		str.WriteString(fsPermStr)
+
+		if fsPermErr != nil {
+			errs = append(errs, fsPermErr)
+		}
+	}
+
+	// Feel free to add additional probes here...
+
+	return str.String(), errs
+}
+
+func ProbeLiveness(ctx context.Context, database *sql.DB, writeablePaths []string, touch string) (string, []error) {
+	// fail immediately if any readiness probes above have already failed.
+	readinessProbeStr, readinessProbeErrs := ProbeReadiness(ctx, database, writeablePaths)
+
+	if len(readinessProbeErrs) != 0 {
+		return readinessProbeStr, readinessProbeErrs
+	}
+
+	var str strings.Builder
+
+	// include previous readiness probe results in final string
+	str.WriteString(readinessProbeStr)
+
+	// slice collects all errors from probes
+	errs := make([]error, 0, 1+len(writeablePaths))
+
+	// DB writeable?
+	dbHealthStr, dbHealthErr := probeDatabaseNextHealthSequence(ctx, database)
+	str.WriteString(dbHealthStr)
+
+	if dbHealthErr != nil {
+		errs = append(errs, dbHealthErr)
+	}
+
+	// FS writeable?
+	for _, writeablePath := range writeablePaths {
+		fsTouchStr, fsTouchErr := probePathWriteableTouch(ctx, writeablePath, touch)
+		str.WriteString(fsTouchStr)
+		if fsTouchErr != nil {
+			errs = append(errs, fsTouchErr)
+		}
+	}
+
+	// Feel free to add additional probes here...
+
+	return str.String(), errs
+}
+
+// FS (especially hard mounted NFS paths) or PostgreSQL calls may be blocking or running for too long and thus need to run detached
+// We additionally want them to timeout (e.g. useful for hard mounted NFS paths)
+// Typically a any context used here will already have a deadline associated
+// If not we will explicitly return a short one here.
+func ensureProbeDeadlineFromContext(ctx context.Context) time.Time {
+	ctxDeadline, hasDeadline := ctx.Deadline()
+	if !hasDeadline {
+		ctxDeadline = time.Now().Add(1 * time.Second)
+	}
+
+	return ctxDeadline
+}
+
+func probeDatabasePingable(ctx context.Context, database *sql.DB) (string, error) {
+	var str strings.Builder
+	ctxDeadline := ensureProbeDeadlineFromContext(ctx)
+
+	dbPingStart := time.Now()
+
+	var dbPingWg sync.WaitGroup
+	var dbErr error
+
+	dbPingWg.Add(1)
+	go func() {
+		dbErr = database.PingContext(ctx)
+		dbPingWg.Done()
+	}()
+
+	if err := util.WaitTimeout(&dbPingWg, time.Until(ctxDeadline)); err != nil {
+		fmt.Fprintf(&str, "Probe db: Ping deadline after %s, error=%v.\n", time.Since(dbPingStart), err.Error())
+		return str.String(), err
+	}
+
+	if dbErr != nil {
+		fmt.Fprintf(&str, "Probe db: Ping errored after %s, error=%v.\n", time.Since(dbPingStart), dbErr.Error())
+		return str.String(), dbErr
+	}
+
+	fmt.Fprintf(&str, "Probe db: Ping succeeded in %s.\n", time.Since(dbPingStart))
+
+	return str.String(), nil
+}
+
+func probeDatabaseNextHealthSequence(ctx context.Context, database *sql.DB) (string, error) {
+	var str strings.Builder
+	ctxDeadline := ensureProbeDeadlineFromContext(ctx)
+
+	dbWriteStart := time.Now()
+
+	var seqVal int
+	var dbWriteWg sync.WaitGroup
+	var dbErr error
+
+	dbWriteWg.Add(1)
+	go func() {
+		dbErr = database.QueryRowContext(ctx, "SELECT nextval('seq_health');").Scan(&seqVal)
+		dbWriteWg.Done()
+	}()
+
+	if err := util.WaitTimeout(&dbWriteWg, time.Until(ctxDeadline)); err != nil {
+		fmt.Fprintf(&str, "Probe db: Next health sequence deadline after %s, error=%v.\n", time.Since(dbWriteStart), err.Error())
+		return str.String(), err
+	}
+
+	if dbErr != nil {
+		fmt.Fprintf(&str, "Probe db: Next health sequence errored after %s, error=%v.\n", time.Since(dbWriteStart), dbErr.Error())
+		return str.String(), dbErr
+	}
+
+	fmt.Fprintf(&str, "Probe db: Next health sequence succeeded in %s, seq_health=%v.\n", time.Since(dbWriteStart), seqVal)
+
+	return str.String(), nil
+}
+
+func probePathWriteablePermission(ctx context.Context, writeablePath string) (string, error) {
+	var str strings.Builder
+	ctxDeadline := ensureProbeDeadlineFromContext(ctx)
+
+	fsWriteStart := time.Now()
+
+	if ctx.Err() != nil {
+		fmt.Fprintf(&str, "Probe path '%s': W_OK check cancelled after %s, error=%v.\n", writeablePath, time.Since(fsWriteStart), ctx.Err())
+		return str.String(), ctx.Err()
+	}
+
+	var fsWriteWg sync.WaitGroup
+	var fsWriteErr error
+	fsWriteWg.Add(1)
+	go func(wp string) {
+		fsWriteErr = unix.Access(wp, unix.W_OK)
+		fsWriteWg.Done()
+	}(writeablePath)
+
+	if err := util.WaitTimeout(&fsWriteWg, time.Until(ctxDeadline)); err != nil {
+		fmt.Fprintf(&str, "Probe path '%s': W_OK check deadline after %s, error=%v.\n", writeablePath, time.Since(fsWriteStart), err)
+		return str.String(), err
+	}
+
+	if fsWriteErr != nil {
+		fmt.Fprintf(&str, "Probe path '%s': W_OK check errored after %s, error=%v.\n", writeablePath, time.Since(fsWriteStart), fsWriteErr.Error())
+		return str.String(), fsWriteErr
+	}
+
+	fmt.Fprintf(&str, "Probe path '%s': W_OK check succeeded in %s.\n", writeablePath, time.Since(fsWriteStart))
+
+	return str.String(), nil
+}
+
+func probePathWriteableTouch(ctx context.Context, writeablePath string, touch string) (string, error) {
+	var str strings.Builder
+	ctxDeadline := ensureProbeDeadlineFromContext(ctx)
+
+	fsTouchStart := time.Now()
+	fsTouchNameAbs := path.Join(writeablePath, touch)
+
+	if ctx.Err() != nil {
+		fmt.Fprintf(&str, "Probe path '%s': Touch cancelled after %s, error=%v.\n", fsTouchNameAbs, time.Since(fsTouchStart), ctx.Err())
+		return str.String(), ctx.Err()
+	}
+
+	var fsTouchWg sync.WaitGroup
+	var fsTouchErr error
+	var fsTouchModTime time.Time
+	fsTouchWg.Add(1)
+	go func(tn string) {
+		fsTouchModTime, fsTouchErr = util.TouchFile(tn)
+		fsTouchWg.Done()
+	}(fsTouchNameAbs)
+
+	if err := util.WaitTimeout(&fsTouchWg, time.Until(ctxDeadline)); err != nil {
+		fmt.Fprintf(&str, "Probe path '%s': Touch deadline after %s, error=%v.\n", fsTouchNameAbs, time.Since(fsTouchStart), err)
+		return str.String(), err
+	}
+
+	if fsTouchErr != nil {
+		fmt.Fprintf(&str, "Probe path '%s': Touch errored after %s, error=%v.\n", fsTouchNameAbs, time.Since(fsTouchStart), fsTouchErr.Error())
+		return str.String(), fsTouchErr
+	}
+
+	fmt.Fprintf(&str, "Probe path '%s': Touch succeeded in %s, modTime=%v.\n", fsTouchNameAbs, time.Since(fsTouchStart), fsTouchModTime.Unix())
+
+	return str.String(), nil
+}

internal/api/handlers/common/probes_internal_test.go

@@ -0,0 +1,70 @@
+// nolint:revive
+package common
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"allaboutapps.dev/aw/go-starter/internal/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEnsureDeadline(t *testing.T) {
+	deadline := time.Now().Add(1 * time.Second)
+
+	ctx, cancel := context.WithDeadline(t.Context(), deadline)
+	defer cancel()
+
+	receivedDeadline := ensureProbeDeadlineFromContext(ctx)
+	assert.Equal(t, deadline, receivedDeadline)
+}
+
+func TestDummyDeadlineWithinOneSec(t *testing.T) {
+	ctx := t.Context()
+
+	receivedDeadline := ensureProbeDeadlineFromContext(ctx)
+	assert.WithinDuration(t, time.Now().Add(1*time.Second), receivedDeadline, 100*time.Millisecond)
+}
+
+func TestProbeDatabasePingableDeadline(t *testing.T) {
+	ctx, cancel := context.WithDeadline(t.Context(), time.Now())
+	defer cancel()
+
+	_, err := probeDatabasePingable(ctx, &sql.DB{})
+	assert.Truef(t, errors.Is(err, util.ErrWaitTimeout) || errors.Is(err, context.DeadlineExceeded), "err must be util.ErrWaitTimeout or context.DeadlineExceeded but is %v", err)
+}
+
+func TestProbeDatabaseNextHealthSequenceDeadline(t *testing.T) {
+	ctx, cancel := context.WithDeadline(t.Context(), time.Now())
+	defer cancel()
+
+	_, err := probeDatabaseNextHealthSequence(ctx, &sql.DB{})
+	assert.Truef(t, errors.Is(err, util.ErrWaitTimeout) || errors.Is(err, context.DeadlineExceeded), "err must be util.ErrWaitTimeout or context.DeadlineExceeded but is %v", err)
+}
+
+func TestProbePathWriteablePermissionContextDeadline(t *testing.T) {
+	ctx, cancel := context.WithDeadline(t.Context(), time.Now())
+	defer cancel()
+
+	_, err := probePathWriteablePermission(ctx, "/any/thing")
+	assert.Truef(t, errors.Is(err, util.ErrWaitTimeout) || errors.Is(err, context.DeadlineExceeded), "err must be util.ErrWaitTimeout or context.DeadlineExceeded but is %v", err)
+}
+
+func TestProbePathWriteableTouchContextDeadline(t *testing.T) {
+	ctx, cancel := context.WithDeadline(t.Context(), time.Now())
+	defer cancel()
+
+	_, err := probePathWriteableTouch(ctx, "/any/thing", ".touch")
+	assert.Truef(t, errors.Is(err, util.ErrWaitTimeout) || errors.Is(err, context.DeadlineExceeded), "err must be util.ErrWaitTimeout or context.DeadlineExceeded but is %v", err)
+}
+
+func TestProbePathWriteableTouchInaccessable(t *testing.T) {
+	_, err := probePathWriteableTouch(t.Context(), "/this/path/does/not/exist", ".touch")
+	require.Error(t, err)
+	assert.ErrorIs(t, err, os.ErrNotExist)
+}