Refactor and add cache and range support.

conf/cache.go

@@ -0,0 +1,7 @@
+package conf
+
+type CacheSettingsYaml struct {
+	MaxAge                               uint `yaml:"maxAge"`
+	NotModifiedResponseUsingLastModified bool `yaml:"notModifiedUsingLastModified"`
+	NotModifiedResponseUsingETags        bool `yaml:"notModifiedUsingETags"`
+}

conf/zone.go

@@ -13,6 +13,8 @@ type ZoneYaml struct {
 	BasePrefixURL      string            `yaml:"basePrefixURL"`
 	SuffixDirectoryURL string            `yaml:"suffixDirectoryURL"`
 	SuffixFileURL      string            `yaml:"suffixFileURL"`
+	RangeSupported     bool              `yaml:"rangeSupported"`
+	CacheSettings      CacheSettingsYaml `yaml:"cacheSettings"`
 }
 
 func (zy ZoneYaml) GetPackageMetaTagOutputter() *outputMeta.PackageMetaTagOutputter {

config.example.yml

@@ -16,3 +16,9 @@ zones: #An array of zones
     username: "captain-alm" #The username to append to the start of a path under the prefix
     suffixDirectoryURL: "src/branch/master{/dir}" #The suffix location of the main branch for directory usage
     suffixFileURL: "src/branch/master{/dir}/{file}#L{line}" #The suffix location of the main branch for file usage
+    rangeSupported: true #Are range requests supported
+    cacheSettings: #Cache settings
+      maxAge: 0 #The maximum age of the cache
+      notModifiedUsingLastModified: true #Are the conditional headers attached to Last-Modified used to work out if to send a 304 Cache Redirect
+      notModifiedUsingETags: true #Are the conditional headers attached to ETag used to work out if to send a 304 Cache Redirect
+

web/page-handler.go

@@ -2,23 +2,33 @@ package web
 
 import (
 	_ "embed"
+	"golang.captainalm.com/GOPackageHeaderServer/conf"
 	"golang.captainalm.com/GOPackageHeaderServer/outputMeta"
+	"golang.captainalm.com/GOPackageHeaderServer/web/utils"
 	"html/template"
+	"io"
+	"mime/multipart"
 	"net/http"
+	"net/textproto"
 	"strconv"
+	"time"
 )
 
 type PageHandler struct {
 	Name           string
 	CSS            string
 	OutputPage     bool
+	RangeSupported bool
+	CacheSettings  conf.CacheSettingsYaml
 	MetaOutput     *outputMeta.PackageMetaTagOutputter
 }
 
+var startTime = time.Now()
+
 //go:embed output-page.html
 var outputPage string
 
-var pageTemplateFuncMap template.FuncMap = template.FuncMap{
+var pageTemplateFuncMap = template.FuncMap{
 	"isNotEmpty": func(stringIn string) bool {
 		return stringIn != ""
 	},
@@ -28,30 +38,60 @@ func (pgh *PageHandler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
 	if request.Method == http.MethodGet || request.Method == http.MethodHead {
 		tmpl, err := template.New("page-handler").Funcs(pageTemplateFuncMap).Parse(outputPage)
 		if err != nil {
-			writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure")
+			utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure")
 			return
 		}
 		tm := handlerTemplateMarshal{
 			PageHandler: *pgh,
 			RequestPath: request.URL.Path,
 		}
-		theBuffer := &BufferedWriter{}
+		theBuffer := &utils.BufferedWriter{}
 		err = tmpl.Execute(theBuffer, tm)
 		if err != nil {
-			writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure")
+			utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure")
 			return
 		}
 		writer.Header().Set("Content-Length", strconv.Itoa(len(theBuffer.Data)))
 		writer.Header().Set("Content-Type", "text/html; charset=utf-8")
-		if writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "") {
-			_, _ = writer.Write(theBuffer.Data)
+		utils.SetLastModifiedHeader(writer.Header(), startTime)
+		utils.SetCacheHeaderWithAge(writer.Header(), pgh.CacheSettings.MaxAge, startTime)
+		theETag := utils.GetValueForETagUsingBufferedWriter(theBuffer)
+		writer.Header().Set("ETag", theETag)
+		if utils.ProcessSupportedPreconditionsForNext(writer, request, startTime, theETag, pgh.CacheSettings.NotModifiedResponseUsingLastModified, pgh.CacheSettings.NotModifiedResponseUsingETags) {
+			httpRangeParts := utils.ProcessRangePreconditions(int64(len(theBuffer.Data)), writer, request, startTime, theETag, pgh.RangeSupported)
+			if httpRangeParts != nil {
+				if len(httpRangeParts) <= 1 {
+					var theWriter io.Writer = writer
+					if len(httpRangeParts) == 1 {
+						theWriter = utils.NewPartialRangeWriter(theWriter, httpRangeParts[0])
+					}
+					_, _ = theWriter.Write(theBuffer.Data)
+				} else {
+					multWriter := multipart.NewWriter(writer)
+					writer.Header().Set("Content-Type", "multipart/byteranges; boundary="+multWriter.Boundary())
+					for _, currentPart := range httpRangeParts {
+						mimePart, err := multWriter.CreatePart(textproto.MIMEHeader{
+							"Content-Range": {currentPart.ToField(int64(len(theBuffer.Data)))},
+							"Content-Type":  {"text/plain; charset=utf-8"},
+						})
+						if err != nil {
+							break
+						}
+						_, err = mimePart.Write(theBuffer.Data[currentPart.Start : currentPart.Start+currentPart.Length])
+						if err != nil {
+							break
+						}
+					}
+					_ = multWriter.Close()
+				}
+			}
 		}
 	} else {
 		writer.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead)
 		if request.Method == http.MethodOptions {
-			writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "")
+			utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "")
 		} else {
-			writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "")
+			utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "")
 		}
 	}
 }

web/utils.go

@@ -1,33 +0,0 @@
-package web
-
-import (
-	"net/http"
-	"strconv"
-)
-
-func writeResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool {
-	hasBody := method != http.MethodHead && method != http.MethodOptions
-	if hasBody && message != "" {
-		rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		rw.Header().Set("X-Content-Type-Options", "nosniff")
-		rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2))
-	}
-	rw.WriteHeader(statusCode)
-	if hasBody {
-		if message != "" {
-			_, _ = rw.Write([]byte(message + "\r\n"))
-			return false
-		}
-		return true
-	}
-	return false
-}
-
-type BufferedWriter struct {
-	Data []byte
-}
-
-func (c *BufferedWriter) Write(p []byte) (n int, err error) {
-	c.Data = append(c.Data, p...)
-	return len(p), nil
-}

web/utils/buffered-writer.go

@@ -0,0 +1,23 @@
+package utils
+
+import (
+	"crypto"
+	"encoding/hex"
+)
+
+type BufferedWriter struct {
+	Data []byte
+}
+
+func (c *BufferedWriter) Write(p []byte) (n int, err error) {
+	c.Data = append(c.Data, p...)
+	return len(p), nil
+}
+
+func (c *BufferedWriter) GetHashString() string {
+	theHash := crypto.SHA1.New()
+	_, _ = theHash.Write(c.Data)
+	theSum := theHash.Sum(nil)
+	theHash.Reset()
+	return hex.EncodeToString(theSum)
+}

web/utils/content-range-value.go

@@ -0,0 +1,78 @@
+package utils
+
+import (
+	"strconv"
+	"strings"
+)
+
+type ContentRangeValue struct {
+	Start, Length int64
+}
+
+func (rstrc ContentRangeValue) ToField(maxLength int64) string {
+	return "bytes " + strconv.FormatInt(rstrc.Start, 10) + "-" + strconv.FormatInt(rstrc.Start+rstrc.Length-1, 10) + "/" + strconv.FormatInt(maxLength, 10)
+}
+
+func GetRanges(rangeStringIn string, maxLength int64) []ContentRangeValue {
+	actualRangeString := strings.TrimPrefix(rangeStringIn, "bytes=")
+	if strings.ContainsAny(actualRangeString, ",") {
+		seperated := strings.Split(actualRangeString, ",")
+		toReturn := make([]ContentRangeValue, len(seperated))
+		pos := 0
+		for _, s := range seperated {
+			if cRange, ok := GetRange(s, maxLength); ok {
+				toReturn[pos] = cRange
+				pos += 1
+			}
+		}
+		if pos == 0 {
+			return nil
+		}
+		return toReturn[:pos]
+	}
+	if cRange, ok := GetRange(actualRangeString, maxLength); ok {
+		return []ContentRangeValue{cRange}
+	}
+	return nil
+}
+
+func GetRange(rangePartIn string, maxLength int64) (ContentRangeValue, bool) {
+	before, after, done := strings.Cut(rangePartIn, "-")
+	before = strings.Trim(before, " ")
+	after = strings.Trim(after, " ")
+	if !done {
+		return ContentRangeValue{}, false
+	}
+	var parsedAfter, parsedBefore int64 = -1, -1
+	if after != "" {
+		if parsed, err := strconv.ParseInt(after, 10, 64); err == nil {
+			parsedAfter = parsed
+		} else {
+			return ContentRangeValue{}, false
+		}
+	}
+	if before != "" {
+		if parsed, err := strconv.ParseInt(before, 10, 64); err == nil {
+			parsedBefore = parsed
+		} else {
+			return ContentRangeValue{}, false
+		}
+	}
+	if parsedBefore >= 0 && parsedAfter > parsedBefore && parsedAfter < maxLength {
+		return ContentRangeValue{
+			Start:  parsedBefore,
+			Length: parsedAfter - parsedBefore + 1,
+		}, true
+	} else if parsedAfter < 0 && parsedBefore >= 0 && parsedBefore < maxLength {
+		return ContentRangeValue{
+			Start:  parsedBefore,
+			Length: maxLength - parsedBefore,
+		}, true
+	} else if parsedBefore < 0 && parsedAfter >= 1 && maxLength-parsedAfter >= 0 {
+		return ContentRangeValue{
+			Start:  maxLength - parsedAfter,
+			Length: parsedAfter,
+		}, true
+	}
+	return ContentRangeValue{}, false
+}

web/utils/counting-writer.go

@@ -0,0 +1,10 @@
+package utils
+
+type CountingWriter struct {
+	Length int64
+}
+
+func (c *CountingWriter) Write(p []byte) (n int, err error) {
+	c.Length += int64(len(p))
+	return len(p), nil
+}

web/utils/etag.go

@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"strings"
+)
+
+func GetValueForETagUsingBufferedWriter(bWriter *BufferedWriter) string {
+	return "\"" + bWriter.GetHashString() + "\""
+}
+
+func GetETagValues(stringIn string) []string {
+	if strings.ContainsAny(stringIn, ",") {
+		seperated := strings.Split(stringIn, ",")
+		toReturn := make([]string, len(seperated))
+		pos := 0
+		for _, s := range seperated {
+			cETag := GetETagValue(s)
+			if cETag != "" {
+				toReturn[pos] = cETag
+				pos += 1
+			}
+		}
+		if pos == 0 {
+			return nil
+		}
+		return toReturn[:pos]
+	}
+	toReturn := []string{GetETagValue(stringIn)}
+	if toReturn[0] == "" {
+		return nil
+	}
+	return toReturn
+}
+
+func GetETagValue(stringIn string) string {
+	startIndex := strings.IndexAny(stringIn, "\"") + 1
+	endIndex := strings.LastIndexAny(stringIn, "\"")
+	if endIndex > startIndex {
+		return stringIn[startIndex:endIndex]
+	}
+	return ""
+}

web/utils/partial-range-writer.go

@@ -0,0 +1,47 @@
+package utils
+
+import "io"
+
+func NewPartialRangeWriter(writerIn io.Writer, httpRangeIn ContentRangeValue) io.Writer {
+	return &PartialRangeWriter{
+		passedWriter:       writerIn,
+		passedWriterIndex:  0,
+		httpRange:          httpRangeIn,
+		exclusiveLastIndex: httpRangeIn.Start + httpRangeIn.Length,
+	}
+}
+
+type PartialRangeWriter struct {
+	passedWriter       io.Writer
+	passedWriterIndex  int64
+	exclusiveLastIndex int64
+	httpRange          ContentRangeValue
+}
+
+func (prw *PartialRangeWriter) Write(p []byte) (n int, err error) {
+	var pOffsetIndex int64 = -1
+	if prw.passedWriterIndex >= prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex {
+		pOffsetIndex = 0
+	} else if prw.passedWriterIndex+int64(len(p)) > prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex {
+		pOffsetIndex = prw.httpRange.Start - prw.passedWriterIndex
+		prw.passedWriterIndex += pOffsetIndex
+	} else {
+		prw.passedWriterIndex += int64(len(p))
+	}
+	if pOffsetIndex >= 0 {
+		if prw.passedWriterIndex+(int64(len(p))-pOffsetIndex) <= prw.exclusiveLastIndex {
+			written, err := prw.passedWriter.Write(p[pOffsetIndex:])
+			prw.passedWriterIndex += int64(written)
+			if err != nil {
+				return written, err
+			}
+		} else {
+			written, err := prw.passedWriter.Write(p[pOffsetIndex : prw.exclusiveLastIndex-prw.passedWriterIndex+pOffsetIndex])
+			prw.passedWriterIndex += int64(written)
+			if err != nil {
+				return written, err
+			}
+		}
+	}
+	return n, nil
+}

web/utils/process-preconditions.go

@@ -0,0 +1,146 @@
+package utils
+
+import (
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func ProcessSupportedPreconditionsForNext(rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, noBypassModify bool, noBypassMatch bool) bool {
+	theStrippedETag := GetETagValue(etag)
+	if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-None-Match") != "" {
+		etagVals := GetETagValues(req.Header.Get("If-None-Match"))
+		conditionSuccess := false
+		for _, s := range etagVals {
+			if s == theStrippedETag {
+				conditionSuccess = true
+				break
+			}
+		}
+		if conditionSuccess {
+			WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "")
+			return false
+		}
+	}
+
+	if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-Match") != "" {
+		etagVals := GetETagValues(req.Header.Get("If-Match"))
+		conditionFailed := true
+		for _, s := range etagVals {
+			if s == theStrippedETag {
+				conditionFailed = false
+				break
+			}
+		}
+		if conditionFailed {
+			SwitchToNonCachingHeaders(rw.Header())
+			WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "")
+			return false
+		}
+	}
+
+	if noBypassModify && !modT.IsZero() && req.Header.Get("If-Modified-Since") != "" {
+		parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Modified-Since"))
+		if err == nil && modT.Before(parse) || strings.EqualFold(modT.Format(http.TimeFormat), req.Header.Get("If-Modified-Since")) {
+			WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "")
+			return false
+		}
+	}
+
+	if noBypassModify && !modT.IsZero() && req.Header.Get("If-Unmodified-Since") != "" {
+		parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Unmodified-Since"))
+		if err == nil && modT.After(parse) {
+			SwitchToNonCachingHeaders(rw.Header())
+			WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "")
+			return false
+		}
+	}
+
+	return true
+}
+
+func ProcessRangePreconditions(maxLength int64, rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, supported bool) []ContentRangeValue {
+	canDoRange := supported
+	theStrippedETag := GetETagValue(etag)
+	modTStr := modT.Format(http.TimeFormat)
+
+	if canDoRange {
+		rw.Header().Set("Accept-Ranges", "bytes")
+	}
+
+	if canDoRange && !modT.IsZero() && strings.HasSuffix(req.Header.Get("If-Range"), "GMT") {
+		newModT, err := time.Parse(http.TimeFormat, modTStr)
+		parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Range"))
+		if err == nil && !newModT.Equal(parse) {
+			canDoRange = false
+		}
+	} else if canDoRange && theStrippedETag != "" && req.Header.Get("If-Range") != "" {
+		if GetETagValue(req.Header.Get("If-Range")) != theStrippedETag {
+			canDoRange = false
+		}
+	}
+
+	if canDoRange && strings.HasPrefix(req.Header.Get("Range"), "bytes=") {
+		if theRanges := GetRanges(req.Header.Get("Range"), maxLength); len(theRanges) != 0 {
+			if len(theRanges) == 1 {
+				rw.Header().Set("Content-Length", strconv.FormatInt(theRanges[0].Length, 10))
+				rw.Header().Set("Content-Range", theRanges[0].ToField(maxLength))
+			} else {
+				theSize := GetMultipartLength(theRanges, rw.Header().Get("Content-Type"), maxLength)
+				rw.Header().Set("Content-Length", strconv.FormatInt(theSize, 10))
+			}
+			if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPartialContent, "") {
+				return theRanges
+			} else {
+				return nil
+			}
+		} else {
+			SwitchToNonCachingHeaders(rw.Header())
+			rw.Header().Set("Content-Range", "bytes */"+strconv.FormatInt(maxLength, 10))
+			WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusRequestedRangeNotSatisfiable, "")
+			return nil
+		}
+	}
+	if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") {
+		return make([]ContentRangeValue, 0)
+	}
+	return nil
+}
+
+func GetMultipartLength(parts []ContentRangeValue, contentType string, maxLength int64) int64 {
+	cWriter := &CountingWriter{Length: 0}
+	var returnLength int64 = 0
+	multWriter := multipart.NewWriter(cWriter)
+	for _, currentPart := range parts {
+		_, _ = multWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Range": {currentPart.ToField(maxLength)},
+			"Content-Type":  {contentType},
+		})
+		returnLength += currentPart.Length
+	}
+	_ = multWriter.Close()
+	returnLength += cWriter.Length
+	return returnLength
+}
+
+func WriteResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool {
+	hasBody := method != http.MethodHead && method != http.MethodOptions
+	if hasBody && message != "" {
+		rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		rw.Header().Set("X-Content-Type-Options", "nosniff")
+		rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2))
+		SetNeverCacheHeader(rw.Header())
+	}
+	rw.WriteHeader(statusCode)
+	if hasBody {
+		if message != "" {
+			_, _ = rw.Write([]byte(message + "\r\n"))
+			return false
+		}
+		return true
+	}
+	return false
+}

web/utils/utils.go

@@ -0,0 +1,45 @@
+package utils
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+)
+
+func SetNeverCacheHeader(header http.Header) {
+	header.Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")
+	header.Set("Pragma", "no-cache")
+}
+
+func SetLastModifiedHeader(header http.Header, modTime time.Time) {
+	if !modTime.IsZero() {
+		header.Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))
+	}
+}
+
+func SetCacheHeaderWithAge(header http.Header, maxAge uint, modifiedTime time.Time) {
+	header.Set("Cache-Control", "max-age="+strconv.Itoa(int(maxAge))+", must-revalidate")
+	if maxAge > 0 {
+		checkerSecondsBetween := int64(time.Now().UTC().Sub(modifiedTime.UTC()).Seconds())
+		if checkerSecondsBetween < 0 {
+			checkerSecondsBetween *= -1
+		}
+		header.Set("Age", strconv.FormatUint(uint64(checkerSecondsBetween)%uint64(maxAge), 10))
+	}
+}
+
+func SwitchToNonCachingHeaders(header http.Header) {
+	SetNeverCacheHeader(header)
+	if header.Get("Last-Modified") != "" {
+		header.Del("Last-Modified")
+	}
+	if header.Get("Age") != "" {
+		header.Del("Age")
+	}
+	if header.Get("Expires") != "" {
+		header.Del("Expires")
+	}
+	if header.Get("ETag") != "" {
+		header.Del("ETag")
+	}
+}

web/web.go

@@ -3,6 +3,7 @@ package web
 import (
 	"github.com/gorilla/mux"
 	"golang.captainalm.com/GOPackageHeaderServer/conf"
+	"golang.captainalm.com/GOPackageHeaderServer/web/utils"
 	"log"
 	"net/http"
 	"strings"
@@ -16,7 +17,9 @@ func New(yaml conf.ConfigYaml) (*http.Server, map[string]*PageHandler) {
 			Name:           zc.Name,
 			CSS:            zc.CssURL,
 			OutputPage:     zc.HavePageContents,
+			RangeSupported: zc.RangeSupported,
 			MetaOutput:     zc.GetPackageMetaTagOutputter(),
+			CacheSettings:  zc.CacheSettings,
 		}
 		for _, d := range zc.Domains {
 			ld := strings.ToLower(d)
@@ -56,13 +59,13 @@ func runBackgroundHttp(s *http.Server) {
 
 func domainNotAllowed(rw http.ResponseWriter, req *http.Request) {
 	if req.Method == http.MethodGet || req.Method == http.MethodHead {
-		writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed")
+		utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed")
 	} else {
 		rw.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead)
 		if req.Method == http.MethodOptions {
-			writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "")
+			utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "")
 		} else {
-			writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "")
+			utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "")
 		}
 	}
 }