Refactor and add cache and range support.
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details

This commit is contained in:
Captain ALM 2022-07-12 15:41:50 +01:00
parent 8c67a34250
commit 0d4036d05c
Signed by: alfred
GPG Key ID: 4E4ADD02609997B1
13 changed files with 478 additions and 62 deletions

7
conf/cache.go Normal file
View File

@ -0,0 +1,7 @@
package conf
type CacheSettingsYaml struct {
MaxAge uint `yaml:"maxAge"`
NotModifiedResponseUsingLastModified bool `yaml:"notModifiedUsingLastModified"`
NotModifiedResponseUsingETags bool `yaml:"notModifiedUsingETags"`
}

View File

@ -3,16 +3,18 @@ package conf
import "golang.captainalm.com/GOPackageHeaderServer/outputMeta"
type ZoneYaml struct {
Name string `yaml:"name"`
Domains []string `yaml:"domains"`
CssURL string `yaml:"cssURL"`
HavePageContents bool `yaml:"havePageContents"`
BasePath string `yaml:"basePath"`
UsernameProvided bool `yaml:"usernameProvided"`
Username string `yaml:"username"`
BasePrefixURL string `yaml:"basePrefixURL"`
SuffixDirectoryURL string `yaml:"suffixDirectoryURL"`
SuffixFileURL string `yaml:"suffixFileURL"`
Name string `yaml:"name"`
Domains []string `yaml:"domains"`
CssURL string `yaml:"cssURL"`
HavePageContents bool `yaml:"havePageContents"`
BasePath string `yaml:"basePath"`
UsernameProvided bool `yaml:"usernameProvided"`
Username string `yaml:"username"`
BasePrefixURL string `yaml:"basePrefixURL"`
SuffixDirectoryURL string `yaml:"suffixDirectoryURL"`
SuffixFileURL string `yaml:"suffixFileURL"`
RangeSupported bool `yaml:"rangeSupported"`
CacheSettings CacheSettingsYaml `yaml:"cacheSettings"`
}
func (zy ZoneYaml) GetPackageMetaTagOutputter() *outputMeta.PackageMetaTagOutputter {

View File

@ -16,3 +16,9 @@ zones: #An array of zones
username: "captain-alm" #The username to append to the start of a path under the prefix
suffixDirectoryURL: "src/branch/master{/dir}" #The suffix location of the main branch for directory usage
suffixFileURL: "src/branch/master{/dir}/{file}#L{line}" #The suffix location of the main branch for file usage
rangeSupported: true #Are range requests supported
cacheSettings: #Cache settings
maxAge: 0 #The maximum age of the cache
notModifiedUsingLastModified: true #Are the conditional headers attached to Last-Modified used to work out if to send a 304 Cache Redirect
notModifiedUsingETags: true #Are the conditional headers attached to ETag used to work out if to send a 304 Cache Redirect

View File

@ -2,23 +2,33 @@ package web
import (
_ "embed"
"golang.captainalm.com/GOPackageHeaderServer/conf"
"golang.captainalm.com/GOPackageHeaderServer/outputMeta"
"golang.captainalm.com/GOPackageHeaderServer/web/utils"
"html/template"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"strconv"
"time"
)
type PageHandler struct {
Name string
CSS string
OutputPage bool
MetaOutput *outputMeta.PackageMetaTagOutputter
Name string
CSS string
OutputPage bool
RangeSupported bool
CacheSettings conf.CacheSettingsYaml
MetaOutput *outputMeta.PackageMetaTagOutputter
}
var startTime = time.Now()
//go:embed output-page.html
var outputPage string
var pageTemplateFuncMap template.FuncMap = template.FuncMap{
var pageTemplateFuncMap = template.FuncMap{
"isNotEmpty": func(stringIn string) bool {
return stringIn != ""
},
@ -28,30 +38,60 @@ func (pgh *PageHandler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
if request.Method == http.MethodGet || request.Method == http.MethodHead {
tmpl, err := template.New("page-handler").Funcs(pageTemplateFuncMap).Parse(outputPage)
if err != nil {
writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure")
utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure")
return
}
tm := handlerTemplateMarshal{
PageHandler: *pgh,
RequestPath: request.URL.Path,
}
theBuffer := &BufferedWriter{}
theBuffer := &utils.BufferedWriter{}
err = tmpl.Execute(theBuffer, tm)
if err != nil {
writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure")
utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure")
return
}
writer.Header().Set("Content-Length", strconv.Itoa(len(theBuffer.Data)))
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
if writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "") {
_, _ = writer.Write(theBuffer.Data)
utils.SetLastModifiedHeader(writer.Header(), startTime)
utils.SetCacheHeaderWithAge(writer.Header(), pgh.CacheSettings.MaxAge, startTime)
theETag := utils.GetValueForETagUsingBufferedWriter(theBuffer)
writer.Header().Set("ETag", theETag)
if utils.ProcessSupportedPreconditionsForNext(writer, request, startTime, theETag, pgh.CacheSettings.NotModifiedResponseUsingLastModified, pgh.CacheSettings.NotModifiedResponseUsingETags) {
httpRangeParts := utils.ProcessRangePreconditions(int64(len(theBuffer.Data)), writer, request, startTime, theETag, pgh.RangeSupported)
if httpRangeParts != nil {
if len(httpRangeParts) <= 1 {
var theWriter io.Writer = writer
if len(httpRangeParts) == 1 {
theWriter = utils.NewPartialRangeWriter(theWriter, httpRangeParts[0])
}
_, _ = theWriter.Write(theBuffer.Data)
} else {
multWriter := multipart.NewWriter(writer)
writer.Header().Set("Content-Type", "multipart/byteranges; boundary="+multWriter.Boundary())
for _, currentPart := range httpRangeParts {
mimePart, err := multWriter.CreatePart(textproto.MIMEHeader{
"Content-Range": {currentPart.ToField(int64(len(theBuffer.Data)))},
"Content-Type": {"text/plain; charset=utf-8"},
})
if err != nil {
break
}
_, err = mimePart.Write(theBuffer.Data[currentPart.Start : currentPart.Start+currentPart.Length])
if err != nil {
break
}
}
_ = multWriter.Close()
}
}
}
} else {
writer.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead)
if request.Method == http.MethodOptions {
writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "")
utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "")
} else {
writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "")
utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "")
}
}
}

View File

@ -1,33 +0,0 @@
package web
import (
"net/http"
"strconv"
)
func writeResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool {
hasBody := method != http.MethodHead && method != http.MethodOptions
if hasBody && message != "" {
rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
rw.Header().Set("X-Content-Type-Options", "nosniff")
rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2))
}
rw.WriteHeader(statusCode)
if hasBody {
if message != "" {
_, _ = rw.Write([]byte(message + "\r\n"))
return false
}
return true
}
return false
}
type BufferedWriter struct {
Data []byte
}
func (c *BufferedWriter) Write(p []byte) (n int, err error) {
c.Data = append(c.Data, p...)
return len(p), nil
}

View File

@ -0,0 +1,23 @@
package utils
import (
"crypto"
"encoding/hex"
)
type BufferedWriter struct {
Data []byte
}
func (c *BufferedWriter) Write(p []byte) (n int, err error) {
c.Data = append(c.Data, p...)
return len(p), nil
}
func (c *BufferedWriter) GetHashString() string {
theHash := crypto.SHA1.New()
_, _ = theHash.Write(c.Data)
theSum := theHash.Sum(nil)
theHash.Reset()
return hex.EncodeToString(theSum)
}

View File

@ -0,0 +1,78 @@
package utils
import (
"strconv"
"strings"
)
type ContentRangeValue struct {
Start, Length int64
}
func (rstrc ContentRangeValue) ToField(maxLength int64) string {
return "bytes " + strconv.FormatInt(rstrc.Start, 10) + "-" + strconv.FormatInt(rstrc.Start+rstrc.Length-1, 10) + "/" + strconv.FormatInt(maxLength, 10)
}
func GetRanges(rangeStringIn string, maxLength int64) []ContentRangeValue {
actualRangeString := strings.TrimPrefix(rangeStringIn, "bytes=")
if strings.ContainsAny(actualRangeString, ",") {
seperated := strings.Split(actualRangeString, ",")
toReturn := make([]ContentRangeValue, len(seperated))
pos := 0
for _, s := range seperated {
if cRange, ok := GetRange(s, maxLength); ok {
toReturn[pos] = cRange
pos += 1
}
}
if pos == 0 {
return nil
}
return toReturn[:pos]
}
if cRange, ok := GetRange(actualRangeString, maxLength); ok {
return []ContentRangeValue{cRange}
}
return nil
}
func GetRange(rangePartIn string, maxLength int64) (ContentRangeValue, bool) {
before, after, done := strings.Cut(rangePartIn, "-")
before = strings.Trim(before, " ")
after = strings.Trim(after, " ")
if !done {
return ContentRangeValue{}, false
}
var parsedAfter, parsedBefore int64 = -1, -1
if after != "" {
if parsed, err := strconv.ParseInt(after, 10, 64); err == nil {
parsedAfter = parsed
} else {
return ContentRangeValue{}, false
}
}
if before != "" {
if parsed, err := strconv.ParseInt(before, 10, 64); err == nil {
parsedBefore = parsed
} else {
return ContentRangeValue{}, false
}
}
if parsedBefore >= 0 && parsedAfter > parsedBefore && parsedAfter < maxLength {
return ContentRangeValue{
Start: parsedBefore,
Length: parsedAfter - parsedBefore + 1,
}, true
} else if parsedAfter < 0 && parsedBefore >= 0 && parsedBefore < maxLength {
return ContentRangeValue{
Start: parsedBefore,
Length: maxLength - parsedBefore,
}, true
} else if parsedBefore < 0 && parsedAfter >= 1 && maxLength-parsedAfter >= 0 {
return ContentRangeValue{
Start: maxLength - parsedAfter,
Length: parsedAfter,
}, true
}
return ContentRangeValue{}, false
}

View File

@ -0,0 +1,10 @@
package utils
type CountingWriter struct {
Length int64
}
func (c *CountingWriter) Write(p []byte) (n int, err error) {
c.Length += int64(len(p))
return len(p), nil
}

42
web/utils/etag.go Normal file
View File

@ -0,0 +1,42 @@
package utils
import (
"strings"
)
func GetValueForETagUsingBufferedWriter(bWriter *BufferedWriter) string {
return "\"" + bWriter.GetHashString() + "\""
}
func GetETagValues(stringIn string) []string {
if strings.ContainsAny(stringIn, ",") {
seperated := strings.Split(stringIn, ",")
toReturn := make([]string, len(seperated))
pos := 0
for _, s := range seperated {
cETag := GetETagValue(s)
if cETag != "" {
toReturn[pos] = cETag
pos += 1
}
}
if pos == 0 {
return nil
}
return toReturn[:pos]
}
toReturn := []string{GetETagValue(stringIn)}
if toReturn[0] == "" {
return nil
}
return toReturn
}
func GetETagValue(stringIn string) string {
startIndex := strings.IndexAny(stringIn, "\"") + 1
endIndex := strings.LastIndexAny(stringIn, "\"")
if endIndex > startIndex {
return stringIn[startIndex:endIndex]
}
return ""
}

View File

@ -0,0 +1,47 @@
package utils
import "io"
func NewPartialRangeWriter(writerIn io.Writer, httpRangeIn ContentRangeValue) io.Writer {
return &PartialRangeWriter{
passedWriter: writerIn,
passedWriterIndex: 0,
httpRange: httpRangeIn,
exclusiveLastIndex: httpRangeIn.Start + httpRangeIn.Length,
}
}
type PartialRangeWriter struct {
passedWriter io.Writer
passedWriterIndex int64
exclusiveLastIndex int64
httpRange ContentRangeValue
}
func (prw *PartialRangeWriter) Write(p []byte) (n int, err error) {
var pOffsetIndex int64 = -1
if prw.passedWriterIndex >= prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex {
pOffsetIndex = 0
} else if prw.passedWriterIndex+int64(len(p)) > prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex {
pOffsetIndex = prw.httpRange.Start - prw.passedWriterIndex
prw.passedWriterIndex += pOffsetIndex
} else {
prw.passedWriterIndex += int64(len(p))
}
if pOffsetIndex >= 0 {
if prw.passedWriterIndex+(int64(len(p))-pOffsetIndex) <= prw.exclusiveLastIndex {
written, err := prw.passedWriter.Write(p[pOffsetIndex:])
prw.passedWriterIndex += int64(written)
if err != nil {
return written, err
}
} else {
written, err := prw.passedWriter.Write(p[pOffsetIndex : prw.exclusiveLastIndex-prw.passedWriterIndex+pOffsetIndex])
prw.passedWriterIndex += int64(written)
if err != nil {
return written, err
}
}
}
return n, nil
}

View File

@ -0,0 +1,146 @@
package utils
import (
"mime/multipart"
"net/http"
"net/textproto"
"strconv"
"strings"
"time"
)
func ProcessSupportedPreconditionsForNext(rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, noBypassModify bool, noBypassMatch bool) bool {
theStrippedETag := GetETagValue(etag)
if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-None-Match") != "" {
etagVals := GetETagValues(req.Header.Get("If-None-Match"))
conditionSuccess := false
for _, s := range etagVals {
if s == theStrippedETag {
conditionSuccess = true
break
}
}
if conditionSuccess {
WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "")
return false
}
}
if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-Match") != "" {
etagVals := GetETagValues(req.Header.Get("If-Match"))
conditionFailed := true
for _, s := range etagVals {
if s == theStrippedETag {
conditionFailed = false
break
}
}
if conditionFailed {
SwitchToNonCachingHeaders(rw.Header())
WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "")
return false
}
}
if noBypassModify && !modT.IsZero() && req.Header.Get("If-Modified-Since") != "" {
parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Modified-Since"))
if err == nil && modT.Before(parse) || strings.EqualFold(modT.Format(http.TimeFormat), req.Header.Get("If-Modified-Since")) {
WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "")
return false
}
}
if noBypassModify && !modT.IsZero() && req.Header.Get("If-Unmodified-Since") != "" {
parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Unmodified-Since"))
if err == nil && modT.After(parse) {
SwitchToNonCachingHeaders(rw.Header())
WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "")
return false
}
}
return true
}
func ProcessRangePreconditions(maxLength int64, rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, supported bool) []ContentRangeValue {
canDoRange := supported
theStrippedETag := GetETagValue(etag)
modTStr := modT.Format(http.TimeFormat)
if canDoRange {
rw.Header().Set("Accept-Ranges", "bytes")
}
if canDoRange && !modT.IsZero() && strings.HasSuffix(req.Header.Get("If-Range"), "GMT") {
newModT, err := time.Parse(http.TimeFormat, modTStr)
parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Range"))
if err == nil && !newModT.Equal(parse) {
canDoRange = false
}
} else if canDoRange && theStrippedETag != "" && req.Header.Get("If-Range") != "" {
if GetETagValue(req.Header.Get("If-Range")) != theStrippedETag {
canDoRange = false
}
}
if canDoRange && strings.HasPrefix(req.Header.Get("Range"), "bytes=") {
if theRanges := GetRanges(req.Header.Get("Range"), maxLength); len(theRanges) != 0 {
if len(theRanges) == 1 {
rw.Header().Set("Content-Length", strconv.FormatInt(theRanges[0].Length, 10))
rw.Header().Set("Content-Range", theRanges[0].ToField(maxLength))
} else {
theSize := GetMultipartLength(theRanges, rw.Header().Get("Content-Type"), maxLength)
rw.Header().Set("Content-Length", strconv.FormatInt(theSize, 10))
}
if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPartialContent, "") {
return theRanges
} else {
return nil
}
} else {
SwitchToNonCachingHeaders(rw.Header())
rw.Header().Set("Content-Range", "bytes */"+strconv.FormatInt(maxLength, 10))
WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusRequestedRangeNotSatisfiable, "")
return nil
}
}
if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") {
return make([]ContentRangeValue, 0)
}
return nil
}
func GetMultipartLength(parts []ContentRangeValue, contentType string, maxLength int64) int64 {
cWriter := &CountingWriter{Length: 0}
var returnLength int64 = 0
multWriter := multipart.NewWriter(cWriter)
for _, currentPart := range parts {
_, _ = multWriter.CreatePart(textproto.MIMEHeader{
"Content-Range": {currentPart.ToField(maxLength)},
"Content-Type": {contentType},
})
returnLength += currentPart.Length
}
_ = multWriter.Close()
returnLength += cWriter.Length
return returnLength
}
func WriteResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool {
hasBody := method != http.MethodHead && method != http.MethodOptions
if hasBody && message != "" {
rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
rw.Header().Set("X-Content-Type-Options", "nosniff")
rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2))
SetNeverCacheHeader(rw.Header())
}
rw.WriteHeader(statusCode)
if hasBody {
if message != "" {
_, _ = rw.Write([]byte(message + "\r\n"))
return false
}
return true
}
return false
}

45
web/utils/utils.go Normal file
View File

@ -0,0 +1,45 @@
package utils
import (
"net/http"
"strconv"
"time"
)
func SetNeverCacheHeader(header http.Header) {
header.Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")
header.Set("Pragma", "no-cache")
}
func SetLastModifiedHeader(header http.Header, modTime time.Time) {
if !modTime.IsZero() {
header.Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))
}
}
func SetCacheHeaderWithAge(header http.Header, maxAge uint, modifiedTime time.Time) {
header.Set("Cache-Control", "max-age="+strconv.Itoa(int(maxAge))+", must-revalidate")
if maxAge > 0 {
checkerSecondsBetween := int64(time.Now().UTC().Sub(modifiedTime.UTC()).Seconds())
if checkerSecondsBetween < 0 {
checkerSecondsBetween *= -1
}
header.Set("Age", strconv.FormatUint(uint64(checkerSecondsBetween)%uint64(maxAge), 10))
}
}
func SwitchToNonCachingHeaders(header http.Header) {
SetNeverCacheHeader(header)
if header.Get("Last-Modified") != "" {
header.Del("Last-Modified")
}
if header.Get("Age") != "" {
header.Del("Age")
}
if header.Get("Expires") != "" {
header.Del("Expires")
}
if header.Get("ETag") != "" {
header.Del("ETag")
}
}

View File

@ -3,6 +3,7 @@ package web
import (
"github.com/gorilla/mux"
"golang.captainalm.com/GOPackageHeaderServer/conf"
"golang.captainalm.com/GOPackageHeaderServer/web/utils"
"log"
"net/http"
"strings"
@ -13,10 +14,12 @@ func New(yaml conf.ConfigYaml) (*http.Server, map[string]*PageHandler) {
var pages = make(map[string]*PageHandler)
for _, zc := range yaml.Zones {
currentPage := &PageHandler{
Name: zc.Name,
CSS: zc.CssURL,
OutputPage: zc.HavePageContents,
MetaOutput: zc.GetPackageMetaTagOutputter(),
Name: zc.Name,
CSS: zc.CssURL,
OutputPage: zc.HavePageContents,
RangeSupported: zc.RangeSupported,
MetaOutput: zc.GetPackageMetaTagOutputter(),
CacheSettings: zc.CacheSettings,
}
for _, d := range zc.Domains {
ld := strings.ToLower(d)
@ -56,13 +59,13 @@ func runBackgroundHttp(s *http.Server) {
func domainNotAllowed(rw http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodGet || req.Method == http.MethodHead {
writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed")
utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed")
} else {
rw.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead)
if req.Method == http.MethodOptions {
writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "")
utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "")
} else {
writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "")
utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "")
}
}
}