================================================
FILE: assets/index.tmpl
================================================
{{ template "header" . }}
some debiman installation
You’re looking at a complete repository of all manpages contained in
Debian.
There are a couple of different ways to use this
repository:
-
-
In your browser address bar, type enough characters of manpages.debian.org,
press TAB, enter the manpage name, hit ENTER.
-
Navigate to the manpage’s address, using this URL schema:
/<suite>/<binarypackage>/<manpage>.<section>.<language>.html
Any part (except <manpage>) can be omitted, and you will be redirected according to our best guess.
-
Browse the repository index:
{{ template "footer" . }}
================================================
FILE: assets/manpage.tmpl
================================================
{{ template "header" . }}
{{ if gt (len .Langs) 1 }}
{{ end }}
{{ if gt (len .Sections) 1 }}
{{ end }}
{{ if gt (len .Bins) 1 }}
{{ end }}
{{ template "footer" . }}
================================================
FILE: assets/manpageerror.tmpl
================================================
{{ template "header" . }}
{{ if gt (len .Langs) 1 }}
{{ end }}
{{ if gt (len .Sections) 1 }}
{{ end }}
{{ if gt (len .Bins) 1 }}
{{ end }}
Sorry, the manpage could not be rendered!
Error message: {{ .Error }}
{{ template "footer" . }}
================================================
FILE: assets/manpagefooterextra.tmpl
================================================
================================================
FILE: assets/notfound.tmpl
================================================
{{ template "header" . }}
{{ if or (ne .BestChoice.Suite "") (eq .Manpage "index") }}
Sorry, I could not find the specific manpage version you requested! Possibly it is no longer in Debian?
{{ else }}
Sorry, the manpage “{{ .Manpage }}” was not found! Did you spell it correctly?
{{ end }}
{{ if ne .BestChoice.Suite "" }}
Could I maybe offer you the manpage {{ .BestChoice.ServingPath ".html" }} instead?
{{ end }}
{{ template "footer" . }}
================================================
FILE: assets/opensearch.xml
================================================
some debiman installation
some debiman installation
================================================
FILE: assets/pkgindex.tmpl
================================================
{{ template "header" . }}
{{ template "footer" . }}
================================================
FILE: assets/srcpkgindex.tmpl
================================================
{{ template "header" . }}
Manpages of src:{{ .Src }} in Debian {{ .First.Package.Suite }}
{{ template "footer" . }}
================================================
FILE: assets/style.css
================================================
@font-face {
font-family: 'Inconsolata';
src: local('Inconsolata'), url({{ BaseURLPath }}/Inconsolata.woff2) format('woff2'), url({{ BaseURLPath }}/Inconsolata.woff) format('woff');
}
@font-face {
font-family: 'Roboto';
font-style: normal;
font-weight: 400;
src: local('Roboto'), local('Roboto Regular'), local('Roboto-Regular'), url({{ BaseURLPath }}/Roboto-Regular.woff2) format('woff2'), url({{ BaseURLPath }}/Roboto-Regular.woff) format('woff');
}
@font-face {
font-family: 'Roboto';
font-style: normal;
font-weight: 700;
/* TODO: is local('Roboto Bold') really correct? */
src: local('Roboto Bold'), local('Roboto-Bold'), url({{ BaseURLPath }}/Roboto-Bold.woff2) format('woff2'), url({{ BaseURLPath }}/Roboto-Bold.woff) format('woff');
}
body {
color: #000;
background-color: white;
font-family: 'Roboto', sans-serif;
font-size: 100%;
line-height: 1.2;
letter-spacing: 0.15px;
margin: 0;
padding: 0;
}
body > div#header {
padding: 0 10px 0 52px;
}
#logo {
position: absolute;
top: 0;
left: 0;
border-left: 1px solid transparent;
border-right: 1px solid transparent;
border-bottom: 1px solid transparent;
width: 50px;
height: 5.07em;
min-height: 65px;
}
#logo a {
display: block;
height: 100%;
}
#logo img {
margin-top: 5px;
position: absolute;
bottom: 0.3em;
overflow: auto;
border: 0;
}
#header h1 {
font-size: 100%;
margin: 0;
}
p.section {
margin: 0;
padding: 0 5px 0 5px;
font-family: monospace;
font-size: 13px;
line-height: 16px;
color: white;
letter-spacing: 0.08em;
position: absolute;
top: 0px;
left: 52px;
background-color: #c70036;
}
p.section a {
color: white;
text-decoration: none;
}
.hidecss {
display: none;
}
#searchbox {
text-align:left;
line-height: 1;
margin: 0 10px 0 0.5em;
padding: 1px 0 1px 0;
position: absolute;
top: 0;
right: 0;
font-size: .75em;
}
#navbar ul {
margin: 0;
padding: 0;
overflow: hidden;
}
#navbar li {
list-style: none;
float: left;
}
#navbar a {
display: block;
color: #0035c7;
text-decoration: none;
border-left: 1px solid transparent;
border-right: 1px solid transparent;
}
#navbar a:hover
, #navbar a:visited:hover {
text-decoration: underline;
}
a:link {
color: #0035c7;
}
a:visited {
color: #54638c;
}
#breadcrumbs {
line-height: 2;
min-height: 20px;
margin: 0;
padding: 0;
font-size: 0.75em;
border-bottom: 1px solid #d2d3d7;
}
#breadcrumbs:before {
margin-left: 0.5em;
margin-right: 0.5em;
}
#content {
margin: 0 10px 0 52px;
display: flex;
flex-direction: row;
word-wrap: break-word;
}
.paneljump {
background-color: #d70751;
padding: 0.5em;
border-radius: 3px;
margin-right: .5em;
display: none;
}
.paneljump a,
.paneljump a:visited,
.paneljump a:hover,
.paneljump a:focus {
color: white;
}
@media all and (max-width: 800px) {
#content {
flex-direction: column;
margin: 0.5em;
}
.paneljump {
display: block;
}
}
.panels {
display: block;
order: 2;
}
.maincontent {
width: 100%;
max-width: 80ch;
order: 1;
}
body > div#footer {
border: 1px solid #dfdfe0;
border-left: 0;
border-right: 0;
background-color: #f5f6f7;
padding: 1em;
margin: 1em 10px 0 52px;
font-size: 0.75em;
line-height: 1.5em;
}
hr {
border-top: 1px solid #d2d3d7;
border-bottom: 1px solid white;
border-left: 0;
border-right: 0;
margin: 1.4375em 0 1.5em 0;
height: 0;
background-color: #bbb;
}
#content p {
padding-left: 1em;
}
/* from tracker.debian.org */
a, a:hover, a:focus, a:visited {
color: #0530D7;
text-decoration: none;
}
/* Panel styles */
.panel {
padding: 15px;
margin-bottom: 20px;
background-color: #ffffff;
border: 1px solid #dddddd;
border-radius: 4px;
-webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
}
.panel-heading, .panel details {
margin: -15px -15px 0px;
background-color: #d70751;
border-bottom: 1px solid #dddddd;
border-top-right-radius: 3px;
border-top-left-radius: 3px;
}
.panel-heading, .panel summary {
padding: 5px 5px;
font-size: 17.5px;
font-weight: 500;
color: #ffffff;
outline-style: none;
}
.panel summary {
padding-left: 7px;
}
summary, details {
display: block;
}
.panel details ul {
margin: 0;
}
.panel-footer {
padding: 5px 5px;
margin: 15px -15px -15px;
background-color: #f5f5f5;
border-top: 1px solid #dddddd;
border-bottom-right-radius: 3px;
border-bottom-left-radius: 3px;
}
.panel-info {
border-color: #bce8f1;
}
.panel-info .panel-heading {
color: #3a87ad;
background-color: #d9edf7;
border-color: #bce8f1;
}
.list-group {
padding-left: 0;
margin-bottom: 20px;
background-color: #ffffff;
}
.list-group-item {
position: relative;
display: block;
padding: 5px 5px 5px 5px;
margin-bottom: -1px;
border: 1px solid #dddddd;
}
.list-group-item > .list-item-key {
min-width: 27%;
display: inline-block;
}
.list-group-item > .list-item-key.versions-repository {
min-width: 40%;
}
.list-group-item > .list-item-key.versioned-links-version {
min-width: 40%
}
.versioned-links-icon {
margin-right: 2px;
}
.versioned-links-icon a {
color: black;
}
.versioned-links-icon a:hover {
color: blue;
}
.versioned-links-icon-inactive {
opacity: 0.5;
}
.list-group-item:first-child {
border-top-right-radius: 4px;
border-top-left-radius: 4px;
}
.list-group-item:last-child {
margin-bottom: 0;
border-bottom-right-radius: 4px;
border-bottom-left-radius: 4px;
}
.list-group-item-heading {
margin-top: 0;
margin-bottom: 5px;
}
.list-group-item-text {
margin-bottom: 0;
line-height: 1.3;
}
.list-group-item:hover {
background-color: #efefef;
}
.list-group-item.active a {
z-index: 2;
}
.list-group-item.active {
background-color: #efefef;
}
.list-group-flush {
margin: 15px -15px -15px;
}
.panel .list-group-flush {
margin-top: -1px;
}
.list-group-flush .list-group-item {
border-width: 1px 0;
}
.list-group-flush .list-group-item:first-child {
border-top-right-radius: 0;
border-top-left-radius: 0;
}
.list-group-flush .list-group-item:last-child {
border-bottom: 0;
}
/* end of tracker.debian.org */
.panel {
float: right;
clear: right;
font-family: 'Roboto';
min-width: 200px;
}
.toc {
/* Limit the content’s width */
width: 200px;
}
.toc li {
font-size: 98%;
letter-spacing: 0.02em;
display: flex;
}
.otherversions {
/* Limit the content’s width */
width: 200px;
}
.otherversions li,
.otherlangs li {
display: flex;
}
.otherversions a,
.otherlangs a {
flex-shrink: 0;
}
.pkgversion,
.pkgname,
.toc a {
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
}
.pkgversion,
.pkgname {
margin-left: auto;
padding-left: 1em;
}
/* mandoc styles */
.mandoc, .mandoc pre, .mandoc code {
font-family: 'Inconsolata', monospace;
font-size: 1.04rem;
}
.mandoc pre {
white-space: pre-wrap;
}
.mandoc {
margin-right: 45px;
/* Required so that table.head and table.foot can take up 100% of what remains after floating the panels. */
overflow: hidden;
margin-top: .5em;
}
table.head, table.foot {
width: 100%;
}
.head-vol {
text-align: center;
}
.head-rtitle {
text-align: right;
}
/* TODO(later): get rid of .spacer once a new-enough mandoc is in Debian */
.spacer, .Pp {
min-height: 1em;
}
pre {
margin-left: 2em;
}
.anchor {
margin-left: .25em;
visibility: hidden;
}
h1:hover .anchor,
h2:hover .anchor,
h3:hover .anchor,
h4:hover .anchor,
h5:hover .anchor,
h6:hover .anchor {
visibility: visible;
}
h1, h2, h3, h4, h5, h6 {
letter-spacing: .07em;
margin-top: 1.5em;
margin-bottom: .35em;
}
h1 {
font-size: 150%;
}
h2 {
font-size: 125%;
}
@media print {
#header, #footer, .panel, .anchor, .paneljump {
display: none;
}
#content {
margin: 0;
}
.mandoc {
margin: 0;
}
}
/* from mandoc.css */
/* Displays and lists. */
.Bd { }
.Bd-indent { margin-left: 3.8em; }
.Bl-bullet { list-style-type: disc;
padding-left: 1em; }
.Bl-bullet > li { }
.Bl-dash { list-style-type: none;
padding-left: 0em; }
.Bl-dash > li:before {
content: "\2014 "; }
.Bl-item { list-style-type: none;
padding-left: 0em; }
.Bl-item > li { }
.Bl-compact > li {
margin-top: 0em; }
.Bl-enum { padding-left: 2em; }
.Bl-enum > li { }
.Bl-compact > li {
margin-top: 0em; }
.Bl-diag { }
.Bl-diag > dt {
font-style: normal;
font-weight: bold; }
.Bl-diag > dd {
margin-left: 0em; }
.Bl-hang { }
.Bl-hang > dt { }
.Bl-hang > dd {
margin-left: 5.5em; }
.Bl-inset { }
.Bl-inset > dt { }
.Bl-inset > dd {
margin-left: 0em; }
.Bl-ohang { }
.Bl-ohang > dt { }
.Bl-ohang > dd {
margin-left: 0em; }
.Bl-tag { margin-left: 5.5em; }
.Bl-tag > dt {
float: left;
margin-top: 0em;
margin-left: -5.5em;
padding-right: 1.2em;
vertical-align: top; }
.Bl-tag > dd {
clear: both;
width: 100%;
margin-top: 0em;
margin-left: 0em;
vertical-align: top;
overflow: auto; }
.Bl-compact > dt {
margin-top: 0em; }
.Bl-column { }
.Bl-column > tbody > tr { }
.Bl-column > tbody > tr > td {
margin-top: 1em; }
.Bl-compact > tbody > tr > td {
margin-top: 0em; }
.Rs { font-style: normal;
font-weight: normal; }
.RsA { }
.RsB { font-style: italic;
font-weight: normal; }
.RsC { }
.RsD { }
.RsI { font-style: italic;
font-weight: normal; }
.RsJ { font-style: italic;
font-weight: normal; }
.RsN { }
.RsO { }
.RsP { }
.RsQ { }
.RsR { }
.RsT { text-decoration: underline; }
.RsU { }
.RsV { }
.eqn { }
.tbl { }
.HP { margin-left: 3.8em;
text-indent: -3.8em; }
/* Semantic markup for command line utilities. */
table.Nm { }
code.Nm { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Fl { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Cm { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Ar { font-style: italic;
font-weight: normal; }
.Op { display: inline; }
.Ic { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Ev { font-style: normal;
font-weight: normal;
font-family: monospace; }
.Pa { font-style: italic;
font-weight: normal; }
/* Semantic markup for function libraries. */
.Lb { }
code.In { font-style: normal;
font-weight: bold;
font-family: inherit; }
a.In { }
.Fd { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Ft { font-style: italic;
font-weight: normal; }
.Fn { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Fa { font-style: italic;
font-weight: normal; }
.Vt { font-style: italic;
font-weight: normal; }
.Va { font-style: italic;
font-weight: normal; }
.Dv { font-style: normal;
font-weight: normal;
font-family: monospace; }
.Er { font-style: normal;
font-weight: normal;
font-family: monospace; }
/* Various semantic markup. */
.An { }
.Lk { }
.Mt { }
.Cd { font-style: normal;
font-weight: bold;
font-family: inherit; }
.Ad { font-style: italic;
font-weight: normal; }
.Ms { font-style: normal;
font-weight: bold; }
.St { }
.Ux { }
/* Physical markup. */
.Bf { display: inline; }
.No { font-style: normal;
font-weight: normal; }
.Em { font-style: italic;
font-weight: normal; }
.Sy { font-style: normal;
font-weight: bold; }
.Li { font-style: normal;
font-weight: normal;
font-family: monospace; }
================================================
FILE: bundle.go
================================================
package bundle
//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/srcpkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
================================================
FILE: cmd/debiman/download.go
================================================
package main
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"unicode/utf8"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/recode"
"github.com/Debian/debiman/internal/write"
"pault.ag/go/archive"
"pault.ag/go/debian/control"
"pault.ag/go/debian/deb"
"pault.ag/go/debian/version"
)
// canSkip returns true if the package is present in the same (or a
// newer) version on disk already.
func canSkip(p pkgEntry, vPath string) bool {
v, err := ioutil.ReadFile(vPath)
if err != nil {
return false
}
vCurrent, err := version.Parse(string(v))
if err != nil {
log.Printf("Warning: could not parse current package version from %q: %v", vPath, err)
return false
}
return version.Compare(vCurrent, p.version) >= 0
}
type contentByBinarypkg []*contentEntry
func (p contentByBinarypkg) Len() int { return len(p) }
func (p contentByBinarypkg) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p contentByBinarypkg) Less(i, j int) bool { return p[i].binarypkg < p[j].binarypkg }
// findClosestFile returns a manpage struct for name, if name exists in the same suite.
// TODO(stapelberg): resolve multiple matches: consider dependencies of src
func findClosestFile(logger *log.Logger, p pkgEntry, src, name string, contentByPath map[string][]*contentEntry) string {
logger.Printf("findClosestFile(src=%q, name=%q)", src, name)
c, ok := contentByPath[strings.TrimPrefix(name, "/usr/share/man/")]
if !ok {
return ""
}
// Ensure we only consider choices within the same suite.
filtered := make([]*contentEntry, 0, len(c))
for _, e := range c {
if e.suite != p.suite {
continue
}
filtered = append(filtered, e)
}
c = filtered
// We still have more than one choice. In case the candidate is in
// the same package as the source link, we take it.
if len(c) > 1 {
var last *contentEntry
cnt := 0
for _, e := range c {
if e.binarypkg != p.binarypkg {
continue
}
last = e
if cnt++; cnt > 1 {
break
}
}
if cnt == 1 {
c = []*contentEntry{last}
}
// We can’t make a 100% correct choice, but we can at least
// make a deterministic choice. The user will see the
// conflicting packages in the navigation panel to ultimately
// resolve the situation, if necessary.
sort.Sort(contentByBinarypkg(c))
}
if len(c) == 0 {
return ""
}
m, err := manpage.FromManPath(strings.TrimPrefix(name, "/usr/share/man/"), &manpage.PkgMeta{
Binarypkg: c[0].binarypkg,
Suite: c[0].suite,
})
logger.Printf("parsing %q as man: %v", name, err)
if err == nil {
return m.ServingPath() + ".gz"
}
return ""
}
func findFile(logger *log.Logger, src, name string, contentByPath map[string][]*contentEntry) (string, string, bool) {
// TODO(later): why is "/"+ in front of src necessary?
searchPath := []string{
"/" + filepath.Dir(src), // “.”
// To prefer manpages in the same language, add “..”, e.g.:
// /usr/share/man/fr/man7/bash-builtins.7 references
// man1/bash.1, which should be taken from
// /usr/share/man/fr/man1/bash.1 instead of
// /usr/share/man/man1/bash.1.
"/" + filepath.Dir(src) + "/..",
"/usr/share/man",
}
logger.Printf("searching reference so=%q", name)
for _, search := range searchPath {
var check string
if filepath.IsAbs(name) {
check = filepath.Clean(name)
} else {
check = filepath.Join(search, name)
}
// Some references include the .gz suffix, some don’t.
if !strings.HasSuffix(check, ".gz") {
check = check + ".gz"
}
c, ok := contentByPath[strings.TrimPrefix(check, "/usr/share/man/")]
if !ok {
log.Printf("%q does not exist", check)
continue
}
sort.Sort(contentByBinarypkg(c))
m, err := manpage.FromManPath(strings.TrimPrefix(check, "/usr/share/man/"), &manpage.PkgMeta{
Binarypkg: c[0].binarypkg,
Suite: c[0].suite,
})
logger.Printf("parsing %q as man: %v", check, err)
if err == nil {
return m.ServingPath() + ".gz", "", true
}
// TODO(later): try to resolve this reference intelligently, i.e. consider installability to narrow down the list of candidates. add a testcase with all cases that we have in all Debian suites currently
return c[0].suite + "/" + c[0].binarypkg + "/aux" + check, check, true
}
return name, "", false
}
func soElim(logger *log.Logger, src string, r io.Reader, w io.Writer, contentByPath map[string][]*contentEntry) ([]string, error) {
var refs []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, ".so ") {
fmt.Fprintln(w, line)
continue
}
so := strings.TrimSpace(line[len(".so "):])
resolved, ref, ok := findFile(logger, src, so, contentByPath)
if !ok {
// Omitting .so lines which cannot be found is consistent
// with what man(1) and other online man viewers do.
logger.Printf("WARNING: could not find .so referenced file %q, omitting the .so line", so)
continue
}
fmt.Fprintf(w, ".so %s\n", resolved)
if ref != "" {
refs = append(refs, ref)
}
}
return refs, scanner.Err()
}
func writeManpage(logger *log.Logger, src, dest string, r io.Reader, m *manpage.Meta, contentByPath map[string][]*contentEntry) ([]string, error) {
var refs []string
content, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
if !utf8.Valid(content) {
content, err = ioutil.ReadAll(recode.Reader(bytes.NewReader(content), m.Language))
if err != nil {
return nil, err
}
}
err = write.Atomically(dest, true, func(w io.Writer) error {
var err error
refs, err = soElim(logger, src, bytes.NewReader(content), w, contentByPath)
return err
})
return refs, err
}
func createAlternativesLinks(logger *log.Logger, p pkgEntry, gv globalView) (map[string]bool, error) {
refs := make(map[string]bool)
key := p.suite + "/" + p.binarypkg
if len(gv.alternatives[key]) == 0 {
return nil, nil
}
logger.Printf("creating %d links for binary package %q", len(gv.alternatives[key]), p.binarypkg)
for _, link := range gv.alternatives[key] {
if !strings.HasPrefix(link.from, "/usr/share/man/") {
continue
}
m, err := manpage.FromManPath(strings.TrimPrefix(link.from, "/usr/share/man/"), &manpage.PkgMeta{
Binarypkg: p.binarypkg,
Suite: p.suite,
})
if err != nil {
logger.Printf("WARNING: file name %q (underneath /usr/share/man) cannot be parsed: %v", link.from, err)
continue
}
resolved := link.to
if !strings.HasSuffix(resolved, ".gz") {
resolved = resolved + ".gz"
}
destsp := findClosestFile(logger, p, link.from, resolved, gv.contentByPath)
if destsp == "" {
// Try to extract the resolved file as non-manpage
// file. If the resolved file does not live in this
// package, this will result in a dangling symlink.
refs[resolved] = true
destsp = filepath.Join(filepath.Dir(m.ServingPath()), "aux", resolved)
logger.Printf("WARNING: possibly dangling symlink %q -> %q, setting to %q", link.from, link.to, destsp)
}
// TODO(stapelberg): add a unit test for this entire function
// TODO(stapelberg): ganeti has an interesting twist: their manpages live outside of usr/share/man, and they only have symlinks. in this case, we should extract the file to aux/ and then mangle the symlink dest. problem: manpages actually are in a separate package (ganeti-2.15) and use an absolute symlink (/etc/ganeti/share), which is not shipped with the package.
rel, err := filepath.Rel(filepath.Dir(m.ServingPath()), destsp)
if err != nil {
logger.Printf("WARNING: %v", err)
continue
}
if err := os.MkdirAll(filepath.Dir(m.ServingPath()), 0755); err != nil {
return refs, err
}
if err := os.Symlink(rel, m.ServingPath()+".gz"); err != nil {
if os.IsExist(err) {
continue
}
return refs, err
}
}
return refs, nil
}
func downloadPkg(ar *archive.Downloader, p pkgEntry, gv globalView) error {
vPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "VERSION")
logger := log.New(os.Stderr, p.suite+"/"+p.binarypkg+": ", log.LstdFlags)
if !*forceReextract && canSkip(p, vPath) {
// Even when skipping the package, the alternatives data we get from
// piuparts might have changed, see issue #119.
if _, err := createAlternativesLinks(logger, p, gv); err != nil {
return err
}
return nil
}
tmp, err := ar.TempFile(control.FileHash{
Filename: p.filename,
Algorithm: "sha256",
Hash: fmt.Sprintf("%x", p.sha256),
})
if err != nil {
return fmt.Errorf("archive download: %v", err)
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
return err
}
allRefs := make(map[string]bool)
d, err := deb.Load(tmp, p.filename)
if err != nil {
return fmt.Errorf("loading %q: %v", p.filename, err)
}
for {
header, err := d.Data.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if header.Typeflag != tar.TypeReg &&
header.Typeflag != tar.TypeRegA &&
header.Typeflag != tar.TypeSymlink &&
header.Typeflag != tar.TypeLink {
continue
}
if header.FileInfo().IsDir() {
continue
}
if !strings.HasPrefix(header.Name, "./usr/share/man/") {
continue
}
destdir := filepath.Join(*servingDir, p.suite, p.binarypkg)
if err := os.MkdirAll(destdir, 0755); err != nil {
return err
}
// TODO: return m?
m, err := manpage.FromManPath(strings.TrimPrefix(header.Name, "./usr/share/man/"), &manpage.PkgMeta{
Binarypkg: p.binarypkg,
Suite: p.suite,
})
if err != nil {
logger.Printf("WARNING: file name %q (underneath /usr/share/man) cannot be parsed: %v", header.Name, err)
continue
}
destPath := filepath.Join(*servingDir, m.ServingPath()+".gz")
if header.Typeflag == tar.TypeLink {
d, err := manpage.FromManPath(strings.TrimPrefix(header.Linkname, "./usr/share/man/"), &manpage.PkgMeta{
Binarypkg: p.binarypkg,
Suite: p.suite,
})
if err != nil {
logger.Printf("WARNING: hard link name %q (underneath /usr/share/man) cannot be parsed: %v", header.Linkname, err)
continue
}
if err := os.Link(filepath.Join(*servingDir, d.ServingPath()+".gz"), m.ServingPath()+".gz"); err != nil {
if os.IsExist(err) {
continue
}
return err
}
continue
}
if header.Typeflag == tar.TypeSymlink {
// filepath.Join calls filepath.Abs
resolved := filepath.Join(filepath.Dir(strings.TrimPrefix(header.Name, ".")), header.Linkname)
if !strings.HasSuffix(resolved, ".gz") {
resolved = resolved + ".gz"
}
destsp := findClosestFile(logger, p, header.Name, resolved, gv.contentByPath)
if destsp == "" {
// Try to extract the resolved file as non-manpage
// file. If the resolved file does not live in this
// package, this will result in a dangling symlink.
allRefs[resolved] = true
destsp = filepath.Join(filepath.Dir(m.ServingPath()), "aux", resolved)
logger.Printf("WARNING: possibly dangling symlink %q -> %q", header.Name, header.Linkname)
}
// TODO(stapelberg): add a unit test for this entire function
// TODO(stapelberg): ganeti has an interesting twist: their manpages live outside of usr/share/man, and they only have symlinks. in this case, we should extract the file to aux/ and then mangle the symlink dest. problem: manpages actually are in a separate package (ganeti-2.15) and use an absolute symlink (/etc/ganeti/share), which is not shipped with the package.
rel, err := filepath.Rel(filepath.Dir(m.ServingPath()), destsp)
if err != nil {
logger.Printf("WARNING: %v", err)
continue
}
if err := os.Symlink(rel, destPath); err != nil {
if os.IsExist(err) {
continue
}
return err
}
if err := maybeSetLinkMtime(destPath, header.ModTime); err != nil {
return err
}
continue
}
r := io.Reader(d.Data)
var gzr *gzip.Reader
if strings.HasSuffix(header.Name, ".gz") {
gzr, err = gzip.NewReader(d.Data)
if err != nil {
return err
}
r = gzr
}
refs, err := writeManpage(logger, header.Name, destPath, r, m, gv.contentByPath)
if err != nil {
return err
}
if err := os.Chtimes(destPath, header.ModTime, header.ModTime); err != nil {
return err
}
if gzr != nil {
if err := gzr.Close(); err != nil {
return err
}
}
for _, r := range refs {
allRefs[r] = true
}
}
// Create all symlinks for slave alternatives.
refs, err := createAlternativesLinks(logger, p, gv)
if err != nil {
return err
}
for r := range refs {
allRefs[r] = true
}
// Extract all non-manpage files which were referenced via .so
// statements, if any.
if len(allRefs) > 0 {
if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
return err
}
d, err = deb.Load(tmp, p.filename)
if err != nil {
return err
}
for {
header, err := d.Data.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if header.Typeflag != tar.TypeReg &&
header.Typeflag != tar.TypeRegA &&
header.Typeflag != tar.TypeSymlink {
continue
}
if header.FileInfo().IsDir() {
continue
}
if !allRefs[strings.TrimPrefix(header.Name, ".")] {
continue
}
destPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "aux", header.Name)
logger.Printf("extracting referenced non-manpage file %q to %q", header.Name, destPath)
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
if err := write.Atomically(destPath, false, func(w io.Writer) error {
_, err := io.Copy(w, d.Data)
return err
}); err != nil {
return err
}
}
}
if err := ioutil.WriteFile(vPath, []byte(p.version.String()), 0644); err != nil {
if os.IsNotExist(err) {
// If the directory does not exist, we did not extract any
// manpages. Since Contents files are not precise (they
// might lag behind), this can happen occasionally.
return nil
}
return fmt.Errorf("Writing version file %q: %v", vPath, err)
}
atomic.AddUint64(&gv.stats.PackagesExtracted, 1)
return nil
}
func parallelDownload(ar *archive.Downloader, gv globalView) error {
eg, ctx := errgroup.WithContext(context.Background())
downloadChan := make(chan pkgEntry)
// TODO: flag for parallelism level
for i := 0; i < 10; i++ {
eg.Go(func() error {
for p := range downloadChan {
if err := downloadPkg(ar, p, gv); err != nil {
return fmt.Errorf("downloading %s/src:%s %v: %v", p.suite, p.source, p.version, err)
}
}
return nil
})
}
for _, p := range gv.pkgs {
select {
case downloadChan <- *p:
case <-ctx.Done():
break
}
}
close(downloadChan)
return eg.Wait()
}
================================================
FILE: cmd/debiman/download_test.go
================================================
package main
import (
"bytes"
"log"
"os"
"strings"
"testing"
)
func TestWriteManpage(t *testing.T) {
table := []struct {
src string
manpage string
want string
wantRefs []string
pkg pkgEntry
contentByPath map[string][]*contentEntry
}{
{
src: "/usr/share/man/man1/noref.1",
manpage: "no ref in here\n",
want: "no ref in here\n",
wantRefs: nil,
pkg: pkgEntry{},
contentByPath: make(map[string][]*contentEntry),
},
{
src: "/usr/share/man/man1/unresolved.1",
manpage: ".so notfound.1\n",
want: "",
wantRefs: nil,
pkg: pkgEntry{},
contentByPath: make(map[string][]*contentEntry),
},
{
src: "/usr/share/man/man1/samepkg.1",
manpage: ".so man1/samepkg.1\n",
want: ".so jessie/bash/samepkg.1.en.gz\n",
wantRefs: nil,
pkg: pkgEntry{
binarypkg: "bash",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{
"man1/samepkg.1.gz": []*contentEntry{
&contentEntry{
binarypkg: "bash",
suite: "jessie",
},
},
},
},
{
src: "/usr/share/man/man1/samepkgaux.1",
manpage: ".so man1/samepkgaux.inc\n",
want: ".so jessie/bash/aux/usr/share/man/man1/samepkgaux.inc.gz\n",
wantRefs: []string{
"/usr/share/man/man1/samepkgaux.inc.gz",
},
pkg: pkgEntry{
binarypkg: "bash",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{
"man1/samepkgaux.inc.gz": []*contentEntry{
&contentEntry{
binarypkg: "bash",
suite: "jessie",
},
},
},
},
{
src: "/usr/share/man/man1/samedir.1",
manpage: ".so samedir.inc\n",
want: ".so jessie/bash/aux/usr/share/man/man1/samedir.inc.gz\n",
wantRefs: []string{
"/usr/share/man/man1/samedir.inc.gz",
},
pkg: pkgEntry{
binarypkg: "bash",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{
"man1/samedir.inc.gz": []*contentEntry{
&contentEntry{
binarypkg: "bash",
suite: "jessie",
},
},
},
},
// example for an absolute path: isdnutils-base/isdnctrl.8.en.gz uses .so /usr/share/man/man8/.isdnctrl_conf.8
{
src: "/usr/share/man/man1/absolute.1",
manpage: ".so /usr/share/man/man8/absolute.8\n",
want: ".so jessie/extra/absolute.8.en.gz\n",
wantRefs: nil,
pkg: pkgEntry{
binarypkg: "bash",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{
"man8/absolute.8.gz": []*contentEntry{
&contentEntry{
binarypkg: "extra",
suite: "jessie",
},
},
},
},
{
src: "/usr/share/man/man1/absolutenotfound.1",
manpage: ".so /usr/share/man/man8/absolute.8\n",
want: "",
wantRefs: nil,
pkg: pkgEntry{
binarypkg: "bash",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{},
},
{
src: "/usr/share/man/fr/man7/bash-builtins.7",
manpage: ".so man1/bash.1\n",
want: ".so jessie/manpages-fr-extra/bash.1.fr.gz\n",
wantRefs: nil,
pkg: pkgEntry{
binarypkg: "manpages-fr-extra",
suite: "jessie",
},
contentByPath: map[string][]*contentEntry{
"man1/bash.1.gz": []*contentEntry{
&contentEntry{
binarypkg: "bash",
suite: "jessie",
},
},
"fr/man1/bash.1.gz": []*contentEntry{
&contentEntry{
binarypkg: "manpages-fr-extra",
suite: "jessie",
},
},
},
},
}
for _, entry := range table {
entry := entry // capture
t.Run(entry.src, func(t *testing.T) {
t.Parallel()
r := strings.NewReader(entry.manpage)
var buf bytes.Buffer
logger := log.New(os.Stderr, "", log.LstdFlags)
refs, err := soElim(logger, entry.src, r, &buf, entry.contentByPath)
if err != nil {
t.Fatal(err)
}
if got, want := buf.String(), entry.want; got != want {
t.Fatalf("Unexpected soElim() result: got %q, want %q", got, want)
}
if got, want := len(refs), len(entry.wantRefs); got != want {
t.Fatalf("Unexpected number of soElim() ref results: got %d, want %d", got, want)
}
for i := 0; i < len(refs); i++ {
if got, want := refs[i], entry.wantRefs[i]; got != want {
t.Fatalf("soElim() ref differs in entry %d: got %q, want %q", i, got, want)
}
}
})
}
}
================================================
FILE: cmd/debiman/getcontents.go
================================================
package main
import (
"bufio"
"bytes"
"fmt"
"io"
"log"
"os"
"golang.org/x/sync/errgroup"
"pault.ag/go/archive"
"pault.ag/go/debian/control"
)
type contentEntry struct {
suite string
arch string
binarypkg string
filename string
}
var manPrefix = []byte("usr/share/man/")
func parseContentsEntry(scanner *bufio.Scanner) ([]*contentEntry, error) {
for scanner.Scan() {
text := scanner.Bytes()
if !bytes.HasPrefix(text, manPrefix) {
continue
}
idx := bytes.LastIndex(text, []byte{' '})
if idx == -1 {
continue
}
parts := bytes.Split(text[idx:], []byte{','})
entries := make([]*contentEntry, 0, len(parts))
for _, part := range parts {
idx2 := bytes.LastIndex(part, []byte{'/'})
if idx2 == -1 {
continue
}
entries = append(entries, &contentEntry{
binarypkg: string(part[idx2+1:]),
filename: string(bytes.TrimSpace(text[len(manPrefix):idx])),
})
}
if len(entries) > 0 {
return entries, nil
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
return nil, io.EOF
}
func getContents(ar *archive.Downloader, suite string, component string, archs []string, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
files := make([]*os.File, len(archs))
scanners := make([]*bufio.Scanner, len(archs))
contents := make([][]*contentEntry, len(archs))
advance := make([]bool, len(archs))
exhausted := make([]bool, len(archs))
var eg errgroup.Group
for idx, arch := range archs {
idx := idx // copy
arch := arch // copy
eg.Go(func() error {
path := component + "/Contents-" + arch + ".gz"
fh, ok := hashByFilename[path]
if !ok {
return fmt.Errorf("ERROR: expected path %q not found in Release file", path)
}
log.Printf("getting %q (hash %v)", suite+"/"+path, fh.Hash)
fh.Filename = "dists/" + suite + "/" + fh.Filename
r, err := ar.TempFile(fh.FileHash)
if err != nil {
return err
}
files[idx] = r
scanners[idx] = bufio.NewScanner(r)
// Some packages have excessively large fields, see e.g.:
// https://bugs.debian.org/942487
scanners[idx].Buffer(nil, 512*1024)
contents[idx], err = parseContentsEntry(scanners[idx])
if err != nil {
if err == io.EOF {
exhausted[idx] = true
return nil
}
return err
}
advance[idx] = false
return nil
})
}
defer func() {
for _, f := range files {
if f != nil {
os.Remove(f.Name())
f.Close()
}
}
}()
if err := eg.Wait(); err != nil {
return nil, err
}
var entries []*contentEntry
for {
for idx, move := range advance {
if !move {
continue
}
var err error
contents[idx], err = parseContentsEntry(scanners[idx])
if err != nil {
if err == io.EOF {
exhausted[idx] = true
} else {
return nil, err
}
}
}
// TODO: unit test for edge cases: can this loop indefinitely or can packages be skipped here?
if done(exhausted) {
break
}
// find the filename which is the least advanced in the sort order
var lowest int
var sum int
for idx := range archs {
sum += len(contents[idx])
if exhausted[idx] {
continue
}
if len(contents[lowest]) == 0 || contents[idx][0].filename < contents[lowest][0].filename {
lowest = idx
}
}
for idx := range advance {
advance[idx] = !exhausted[idx] && contents[lowest][0].filename == contents[idx][0].filename
}
binarypkgs := make(map[string]string, sum)
for idx := range archs {
if !advance[idx] {
continue
}
for _, e := range contents[idx] {
// first arch (amd64) wins
if _, ok := binarypkgs[e.binarypkg]; !ok {
binarypkgs[e.binarypkg] = archs[idx]
}
}
}
for pkg, arch := range binarypkgs {
entries = append(entries, &contentEntry{
binarypkg: pkg,
arch: arch,
filename: contents[lowest][0].filename,
suite: suite,
})
}
}
return entries, nil
}
func getAllContents(ar *archive.Downloader, suite string, release *archive.Release, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
// We skip archAll, because there is no Contents-all file. The
// contents of Architecture: all packages are included in the
// architecture-specific Contents-* files.
var components = [...]string{"main", "contrib"}
parts := make([][]*contentEntry, len(components))
var sum int
for idx, component := range components {
archs := make([]string, len(release.Architectures))
for idx, arch := range release.Architectures {
archs[idx] = arch.String()
}
part, err := getContents(ar, suite, component, archs, hashByFilename)
if err != nil {
return nil, err
}
parts[idx] = part
sum += len(part)
}
results := make([]*contentEntry, 0, sum)
for _, part := range parts {
results = append(results, part...)
}
return results, nil
}
================================================
FILE: cmd/debiman/getpackages.go
================================================
package main
import (
"bufio"
"bytes"
"encoding/hex"
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
"golang.org/x/sync/errgroup"
"github.com/Debian/debiman/internal/manpage"
"pault.ag/go/archive"
"pault.ag/go/debian/control"
"pault.ag/go/debian/version"
)
type pkgEntry struct {
source string
suite string
binarypkg string
arch string
filename string
version version.Version
sha256 []byte
bytes int64
replaces []string
}
// TODO(later): containsMans could be a map[string]bool, if only all
// Debian packages would ship their manpages in all
// architectures. Example of a package which is doing it wrong:
// “inventor-clients”, which only contains manpages in i386.
//
// In theory, /usr/share must contain the same files across
// architectures: the file-system hierarchy standard (FHS) specifies
// that /usr/share is reserved for architecture independent files, see
// http://www.pathname.com/fhs/pub/fhs-2.3.html#USRSHAREARCHITECTUREINDEPENDENTDATA
// TODO(later): find out which packages are affected and file bugs
func buildContainsMains(content []*contentEntry, links map[string][]link) map[string]map[string]bool {
containsMans := make(map[string]map[string]bool)
for _, entry := range content {
if _, ok := containsMans[entry.binarypkg]; !ok {
containsMans[entry.binarypkg] = make(map[string]bool)
}
containsMans[entry.binarypkg][entry.arch] = true
}
for key := range links {
// key is e.g. “testing/vim-nox”
idx := strings.Index(key, "/")
binarypkg := key[idx+1:]
if containsMans[binarypkg] == nil {
containsMans[binarypkg] = map[string]bool{mostPopularArchitecture: true}
}
}
log.Printf("%d content entries, %d packages\n", len(content), len(containsMans))
return containsMans
}
var emptyVersion version.Version
var (
prefixPackage = []byte("Package")
prefixSource = []byte("Source")
prefixVersion = []byte("Version")
prefixFilename = []byte("Filename")
prefixSize = []byte("Size")
prefixSHA256 = []byte("SHA256")
prefixReplaces = []byte("Replaces")
)
func parsePackageParagraph(scanner *bufio.Scanner, arch string, containsMans map[string]map[string]bool) (pkgEntry, error) {
var entry pkgEntry
for scanner.Scan() {
text := scanner.Bytes()
if len(text) == 0 {
entry = pkgEntry{}
continue
}
idx := bytes.IndexByte(text, ':')
if idx == -1 {
continue
}
key := text[:idx]
if bytes.Equal(key, prefixPackage) {
entry.binarypkg = string(text[idx+2:])
} else if bytes.Equal(key, prefixSource) {
entry.source = string(text[idx+2:])
} else if bytes.Equal(key, prefixVersion) {
v, err := version.Parse(string(text[idx+2:]))
if err != nil {
return entry, err
}
entry.version = v
} else if bytes.Equal(key, prefixFilename) {
entry.filename = string(text[idx+2:])
} else if bytes.Equal(key, prefixSize) {
i, err := strconv.ParseInt(string(text[idx+2:]), 0, 64)
if err != nil {
return entry, err
}
entry.bytes = i
} else if bytes.Equal(key, prefixSHA256) {
h := make([]byte, hex.DecodedLen(len(text[idx+2:])))
n, err := hex.Decode(h, text[idx+2:])
if err != nil {
return entry, err
}
entry.sha256 = h[:n]
} else if bytes.Equal(key, prefixReplaces) {
// e.g. Replaces: systemd (<< 224-2)
pkgs := strings.Split(string(text[idx+2:]), ",")
for _, pkg := range pkgs {
if idx := strings.Index(pkg, " "); idx > -1 {
pkg = pkg[:idx]
}
entry.replaces = append(entry.replaces, pkg)
}
}
if entry.binarypkg != "" &&
entry.version != emptyVersion &&
entry.filename != "" &&
entry.bytes > 0 &&
entry.sha256 != nil {
if !containsMans[entry.binarypkg][arch] {
entry = pkgEntry{}
continue
}
if entry.source == "" {
entry.source = entry.binarypkg
}
idx := strings.Index(entry.source, " ")
if idx > -1 {
entry.source = entry.source[:idx]
}
return entry, nil
}
}
if err := scanner.Err(); err != nil {
return entry, err
}
entry = pkgEntry{}
return entry, io.EOF
}
func less(a, b pkgEntry) bool {
if a.source == b.source {
return a.binarypkg < b.binarypkg
}
return a.source < b.source
}
func done(exhausted []bool) bool {
for idx := range exhausted {
if !exhausted[idx] {
return false
}
}
return true
}
func getPackages(ar *archive.Downloader, rd *archive.ReleaseDownloader, suite string, component string, archs []string, hashByFilename map[string]*control.SHA256FileHash, containsMans map[string]map[string]bool) ([]*pkgEntry, map[string]*manpage.PkgMeta, error) {
files := make([]*os.File, len(archs))
scanners := make([]*bufio.Scanner, len(archs))
pkgs := make([]pkgEntry, len(archs))
advance := make([]bool, len(archs))
exhausted := make([]bool, len(archs))
var eg errgroup.Group
for idx, arch := range archs {
idx := idx // copy
arch := arch // copy
eg.Go(func() error {
// Prefer gzip over xz because gzip uncompresses faster.
path := component + "/binary-" + arch + "/Packages.gz"
fh, ok := hashByFilename[path]
if !ok {
path = component + "/binary-" + arch + "/Packages.xz"
fh, ok = hashByFilename[path]
if !ok {
return fmt.Errorf("ERROR: expected path %q not found in Release file", path)
}
}
log.Printf("getting %q (hash %v)", suite+"/"+path, fh.Hash)
r, err := rd.TempFile(fh.FileHash)
if err != nil {
return err
}
files[idx] = r
scanners[idx] = bufio.NewScanner(r)
// Some packages have excessively large fields, see e.g.:
// https://bugs.debian.org/942487
scanners[idx].Buffer(nil, 512*1024)
advance[idx] = true
return nil
})
}
defer func() {
for _, f := range files {
if f != nil {
os.Remove(f.Name())
f.Close()
}
}
}()
if err := eg.Wait(); err != nil {
return nil, nil, err
}
byVersion := make(map[string]*pkgEntry)
for {
for idx, move := range advance {
if !move {
continue
}
arch := archs[idx]
p, err := parsePackageParagraph(scanners[idx], arch, containsMans)
if err != nil {
if err == io.EOF {
exhausted[idx] = true
} else {
return nil, nil, err
}
}
p.arch = arch
p.suite = suite
pkgs[idx] = p
}
// TODO: unit test for edge cases: can this loop indefinitely or can packages be skipped here?
if done(exhausted) {
break
}
// find the package which is the least advanced in the sort order
lowest := -1
for idx := range archs {
if exhausted[idx] {
continue
}
if lowest == -1 || less(pkgs[idx], pkgs[lowest]) {
lowest = idx
}
}
for idx := range advance {
advance[idx] = !exhausted[idx] && !less(pkgs[lowest], pkgs[idx])
}
// find the best architecture for that package
var newest *pkgEntry
for idx := range archs {
if exhausted[idx] {
continue
}
if less(pkgs[lowest], pkgs[idx]) {
continue
}
if newest == nil || version.Compare(pkgs[idx].version, newest.version) > 0 {
newest = &(pkgs[idx])
}
}
key := suite + "/" + newest.binarypkg
if v, ok := byVersion[key]; ok && version.Compare(v.version, newest.version) > 0 {
continue
}
var best *pkgEntry
for idx, p := range pkgs {
if exhausted[idx] {
continue
}
if less(pkgs[lowest], pkgs[idx]) {
continue
}
if p.version != newest.version {
continue
}
if p.arch == mostPopularArchitecture {
best = &(pkgs[idx])
break
}
}
if best == nil {
for idx, p := range pkgs {
if exhausted[idx] {
continue
}
if less(pkgs[lowest], pkgs[idx]) {
continue
}
if p.version != newest.version {
continue
}
best = &(pkgs[idx])
break
}
}
entry := *best // copy
byVersion[key] = &entry
}
result := make([]*pkgEntry, 0, len(byVersion))
latestVersion := make(map[string]*manpage.PkgMeta, len(byVersion))
for key, p := range byVersion {
result = append(result, p)
latestVersion[key] = &manpage.PkgMeta{
Replaces: p.replaces,
Component: component,
Filename: p.filename,
Sourcepkg: p.source,
Binarypkg: p.binarypkg,
Suite: p.suite,
Version: p.version,
}
}
return result, latestVersion, nil
}
func getAllPackages(ar *archive.Downloader, rd *archive.ReleaseDownloader, suite string, release *archive.Release, hashByFilename map[string]*control.SHA256FileHash, containsMans map[string]map[string]bool) ([]*pkgEntry, map[string]*manpage.PkgMeta, error) {
var components = [...]string{"main", "contrib"}
partsp := make([][]*pkgEntry, len(components))
partsl := make([]map[string]*manpage.PkgMeta, len(components))
latestVersion := make(map[string]*manpage.PkgMeta)
var sum int
for idx, component := range components {
archs := make([]string, len(release.Architectures))
for idx, arch := range release.Architectures {
archs[idx] = arch.String()
}
partp, partl, err := getPackages(ar, rd, suite, component, archs, hashByFilename, containsMans)
if err != nil {
return nil, nil, err
}
partsp[idx] = partp
partsl[idx] = partl
sum += len(partp)
}
results := make([]*pkgEntry, 0, sum)
for idx := range partsp {
results = append(results, partsp[idx]...)
for key, value := range partsl[idx] {
latestVersion[key] = value
}
}
return results, latestVersion, nil
}
================================================
FILE: cmd/debiman/globalview.go
================================================
package main
import (
"compress/gzip"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/sync/errgroup"
"github.com/Debian/debiman/internal/manpage"
"pault.ag/go/archive"
"pault.ag/go/debian/control"
)
// mostPopularArchitecture is used as preferred architecture when we
// need to pick an arbitrary architecture. The rationale is that
// downloading the package for the most popular architecture has the
// least bad influence on the mirror server’s caches.
const mostPopularArchitecture = "amd64"
type stats struct {
PackagesExtracted uint64
PackagesDeleted uint64
ManpagesRendered uint64
ManpageBytes uint64
HTMLBytes uint64
IndexBytes uint64
}
type link struct {
from string
to string
}
type globalView struct {
// pkgs contains all binary packages we know of.
pkgs []*pkgEntry
// suites contains the Debian suites that we know of. Can either be a codename or a suite,
// depending on the values of -sync_codenames and -sync_suites.
// e.g. “stretch” (codename) or “stable” (suite)
suites map[string]bool
// idxSuites maps codename, suite and command-line argument to suite (as in
// suites).
// e.g. map[oldoldstable:wheezy wheezy:wheezy]
idxSuites map[string]string
// contentByPath maps from paths underneath /usr/share/man to a contentEntry.
contentByPath map[string][]*contentEntry
// xref maps from manpage.Meta.Name (e.g. “w3m” or “systemd.service”) to
// the corresponding manpage.Meta.
xref map[string][]*manpage.Meta
// alternatives maps from Debian binary package to a slice of
// links (from→to pairs).
alternatives map[string][]link
stats *stats
start time.Time
}
type distributionIdentifier int
const (
fromCodename = iota
fromSuite
)
type distribution struct {
name string
identifier distributionIdentifier
}
// distributions returns a list of all distributions (either codenames
// [e.g. wheezy, jessie] or suites [e.g. testing, unstable]) from the
// -sync_codenames and -sync_suites flags.
func distributions(codenames []string, suites []string) []distribution {
distributions := make([]distribution, 0, len(codenames)+len(suites))
for _, e := range codenames {
e = strings.TrimSpace(e)
if e == "" {
continue
}
distributions = append(distributions, distribution{
name: e,
identifier: fromCodename})
}
for _, e := range suites {
e = strings.TrimSpace(e)
if e == "" {
continue
}
distributions = append(distributions, distribution{
name: e,
identifier: fromSuite})
}
return distributions
}
func parseAlternativesFile(fn, prefix string) (map[string][]link, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
r, err := gzip.NewReader(f)
if err != nil {
return nil, err
}
defer r.Close()
res := make(map[string][]link)
dec := json.NewDecoder(r)
// read open bracket
if _, err := dec.Token(); err != nil {
return nil, err
}
for dec.More() {
var m struct {
Binpackage string
From string
To string
}
if err := dec.Decode(&m); err != nil {
return nil, err
}
log.Printf("adding from %q to %q to pkg %q", m.From, m.To, m.Binpackage)
key := prefix + "/" + m.Binpackage
res[key] = append(res[key], link{
from: m.From,
to: m.To,
})
}
return res, nil
}
func parseAlternativesDir(dir string) (map[string][]link, error) {
if dir == "" {
return map[string][]link{}, nil
}
infos, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
results := make([]map[string][]link, len(infos))
var eg errgroup.Group
for idx, fi := range infos {
idx, fi := idx, fi // copy
eg.Go(func() error {
suite := strings.TrimSuffix(fi.Name(), ".json.gz")
res, err := parseAlternativesFile(filepath.Join(dir, fi.Name()), suite)
results[idx] = res
return err
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
// Merge all subresults into one map. This is non-destructive
// because the keys are prefixed by the Debian suite, which is
// derived from the filename and hence unique.
merged := make(map[string][]link)
for idx := range infos {
for key, val := range results[idx] {
merged[key] = val
}
}
return merged, nil
}
func markPresent(latestVersion map[string]*manpage.PkgMeta, xref map[string][]*manpage.Meta, filename string, key string) error {
if _, ok := latestVersion[key]; !ok {
return fmt.Errorf("Could not determine latest version")
}
m, err := manpage.FromManPath(strings.TrimPrefix(filename, "usr/share/man/"), latestVersion[key])
if err != nil {
return fmt.Errorf("Trying to interpret path %q: %v", filename, err)
}
// NOTE(stapelberg): this additional verification step
// is necessary because manpages such as the French
// manpage for qelectrotech(1) are present in multiple
// encodings. manpageFromManPath ignores encodings, so
// if we didn’t filter, we would end up with what
// looks like duplicates.
present := false
for _, x := range xref[m.Name] {
if x.ServingPath() == m.ServingPath() {
present = true
break
}
}
if !present {
xref[m.Name] = append(xref[m.Name], m)
}
return nil
}
func buildGlobalView(ar *archive.Downloader, dists []distribution, alternativesDir string, start time.Time) (globalView, error) {
var stats stats
res := globalView{
suites: make(map[string]bool, len(dists)),
idxSuites: make(map[string]string, len(dists)),
contentByPath: make(map[string][]*contentEntry),
xref: make(map[string][]*manpage.Meta),
stats: &stats,
start: start,
}
var err error
res.alternatives, err = parseAlternativesDir(alternativesDir)
if err != nil {
return res, err
}
for _, dist := range dists {
release, rd, err := ar.Release(dist.name)
if err != nil {
return res, err
}
var suite string
if dist.identifier == fromCodename {
suite = release.Codename // e.g. “stretch”
} else {
suite = release.Suite // e.g. “stable”
}
res.suites[suite] = true
res.idxSuites[release.Suite] = suite
res.idxSuites[release.Codename] = suite
res.idxSuites[dist.name] = suite
hashByFilename := make(map[string]*control.SHA256FileHash, len(release.SHA256))
for idx, fh := range release.SHA256 {
// fh.Filename contains e.g. “non-free/source/Sources”
hashByFilename[fh.Filename] = &(release.SHA256[idx])
}
content, err := getAllContents(ar, suite, release, hashByFilename)
if err != nil {
return res, err
}
for _, c := range content {
res.contentByPath[c.filename] = append(res.contentByPath[c.filename], c)
}
var latestVersion map[string]*manpage.PkgMeta
{
// Collect package download work units
var pkgs []*pkgEntry
var err error
pkgs, latestVersion, err = getAllPackages(ar, rd, suite, release, hashByFilename, buildContainsMains(content, res.alternatives))
if err != nil {
return res, err
}
log.Printf("Adding %d packages from suite %q", len(pkgs), suite)
res.pkgs = append(res.pkgs, pkgs...)
}
knownIssues := make(map[string][]error)
// Build a global view of all the manpages (required for cross-referencing).
// TODO(issue): edge case: packages which got renamed between releases
for _, c := range content {
key := c.suite + "/" + c.binarypkg
if err := markPresent(latestVersion, res.xref, c.filename, key); err != nil {
knownIssues[key] = append(knownIssues[key], err)
}
}
for key, links := range res.alternatives {
for _, link := range links {
log.Printf("key=%q, link=%v, latest = %v", key, link, latestVersion[key])
if err := markPresent(latestVersion, res.xref, strings.TrimPrefix(link.from, "/"), key); err != nil {
knownIssues[key] = append(knownIssues[key], err)
}
}
}
for key, errors := range knownIssues {
// TODO: write these to a known-issues file, parse bug numbers from an auxiliary file
log.Printf("package %q has errors: %v", key, errors)
}
}
return res, nil
}
================================================
FILE: cmd/debiman/main.go
================================================
package main
import (
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/crypto/openpgp"
_ "net/http/pprof"
"github.com/Debian/debiman/internal/bundled"
"github.com/Debian/debiman/internal/commontmpl"
"github.com/Debian/debiman/internal/write"
"pault.ag/go/archive"
)
var (
servingDir = flag.String("serving_dir",
"/srv/man",
"Directory in which to place the manpages which should be served")
indexPath = flag.String("index",
"
/auxserver.idx",
"Path to an auxserver index to generate")
syncCodenames = flag.String("sync_codenames",
"",
"Debian codenames to synchronize (e.g. wheezy, jessie, …)")
syncSuites = flag.String("sync_suites",
"testing",
"Debian suites to synchronize (e.g. testing, unstable)")
onlyRender = flag.String("only_render_pkgs",
"",
"If non-empty, a comma-separated whitelist of packages to render (for developing)")
forceRerender = flag.Bool("force_rerender",
false,
"Forces all manpages to be re-rendered, even if they are up to date")
forceReextract = flag.Bool("force_reextract",
false,
"Forces all manpages to be re-extracted, even if there is no newer package version")
remoteMirror = flag.String("remote_mirror",
"http://localhost:3142/deb.debian.org/",
"URL of a Debian mirror to fetch packages from. localhost:3142 is provided by apt-cacher-ng")
localMirror = flag.String("local_mirror",
"",
"If non-empty, a file system path to a Debian mirror, e.g. /srv/mirrors/debian on DSA-maintained machines")
injectAssets = flag.String("inject_assets",
"",
"If non-empty, a file system path to a directory containing assets to overwrite")
alternativesDir = flag.String("alternatives_dir",
"",
"If non-empty, a directory containing JSON-encoded lists of slave alternative links, named after the suite (e.g. sid.json.gz, testing.json.gz, etc.)")
keyring = flag.String("keyring",
"",
"If non-empty, the specified GPG public keyring will be used for validating archive signatures instead of "+archive.DebianArchiveKeyring)
showVersion = flag.Bool("version",
false,
"Show debiman version and exit")
)
// use go build -ldflags "-X main.debimanVersion=" to set the version
var debimanVersion = "HEAD"
// TODO: handle deleted packages, i.e. packages which are present on
// disk but not in pkgs
// TODO(later): add memory usage estimates to the big structures, set
// parallelism level according to available memory on the system
func logic() error {
start := time.Now()
ar := &archive.Downloader{
Parallel: 10,
MaxTransientRetries: 3,
Mirror: *remoteMirror + "/debian",
LocalMirror: *localMirror,
}
if *keyring != "" {
f, err := os.Open(*keyring)
if err != nil {
return fmt.Errorf("loading -keyring: %v", err)
}
defer f.Close()
ar.Keyring, err = openpgp.ReadKeyRing(f)
if err != nil {
return fmt.Errorf("ReadKeyRing(%s): %v", *keyring, err)
}
}
// Stage 1: all Debian packages of all architectures of the
// specified suites are discovered.
globalView, err := buildGlobalView(ar, distributions(
strings.Split(*syncCodenames, ","),
strings.Split(*syncSuites, ",")),
*alternativesDir,
start)
if err != nil {
return fmt.Errorf("gathering packages: %v", err)
}
log.Printf("gathered packages of all suites, total %d packages", len(globalView.pkgs))
// Stage 2: man pages and auxiliary files (e.g. content fragment
// files which are included by a number of manpages) are extracted
// from the identified Debian packages.
if err := parallelDownload(ar, globalView); err != nil {
return fmt.Errorf("extracting manpages: %v", err)
}
log.Printf("Extracted all manpages, now rendering")
// Stage 3: all man pages are rendered into an HTML representation
// using mandoc(1), directory index files are rendered, contents
// files are rendered.
if err := renderAll(globalView); err != nil {
return fmt.Errorf("rendering manpages: %v", err)
}
log.Printf("Rendered all manpages, writing index")
// Stage 4: write the index only after all rendering is complete,
// otherwise debiman-auxserver might serve redirects to pages
// which cannot be served yet.
path := strings.Replace(*indexPath, "", *servingDir, -1)
log.Printf("Writing debiman-auxserver index to %q", path)
if err := writeIndex(path, globalView); err != nil {
return fmt.Errorf("writing index: %v", err)
}
if err := renderAux(*servingDir, globalView); err != nil {
return fmt.Errorf("rendering aux files: %v", err)
}
fmt.Printf("total number of packages: %d\n", len(globalView.pkgs))
fmt.Printf("packages extracted: %d\n", globalView.stats.PackagesExtracted)
fmt.Printf("packages deleted: %d\n", globalView.stats.PackagesDeleted)
fmt.Printf("manpages rendered: %d\n", globalView.stats.ManpagesRendered)
fmt.Printf("total manpage bytes: %d\n", globalView.stats.ManpageBytes)
fmt.Printf("total HTML bytes: %d\n", globalView.stats.HTMLBytes)
fmt.Printf("auxserver index bytes: %d\n", globalView.stats.IndexBytes)
fmt.Printf("wall-clock runtime (s): %d\n", int(time.Now().Sub(start).Seconds()))
return write.Atomically(filepath.Join(*servingDir, "metrics.txt"), false, func(w io.Writer) error {
if err := writeMetrics(w, globalView, start); err != nil {
return fmt.Errorf("writing metrics: %v", err)
}
return nil
})
}
func main() {
flag.Parse()
log.SetFlags(log.LstdFlags | log.Lshortfile)
if *showVersion {
fmt.Printf("debiman %s\n", debimanVersion)
return
}
if *injectAssets != "" {
if err := bundled.Inject(*injectAssets); err != nil {
log.Fatal(err)
}
commonTmpls = commontmpl.MustParseCommonTmpls()
contentsTmpl = mustParseContentsTmpl()
pkgindexTmpl = mustParsePkgindexTmpl()
srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
indexTmpl = mustParseIndexTmpl()
faqTmpl = mustParseFaqTmpl()
aboutTmpl = mustParseAboutTmpl()
manpageTmpl = mustParseManpageTmpl()
manpageerrorTmpl = mustParseManpageerrorTmpl()
manpagefooterextraTmpl = mustParseManpagefooterextraTmpl()
}
// All of our .so references are relative to *servingDir. For
// mandoc(1) to find the files, we need to change the working
// directory now.
//
// We turn *servingDir into an absolute path so that it still refers to the
// same location even after our os.Chdir() call (see issue #152).
abs, err := filepath.Abs(*servingDir)
if err != nil {
log.Fatal(err)
}
*servingDir = abs
if err := os.Chdir(*servingDir); err != nil {
log.Fatal(err)
}
go http.ListenAndServe(":4414", nil)
if err := logic(); err != nil {
log.Fatal(err)
}
}
================================================
FILE: cmd/debiman/main_test.go
================================================
package main
import (
"flag"
"io/ioutil"
"os"
"testing"
)
func TestEndToEnd(t *testing.T) {
dir, err := ioutil.TempDir("", "debiman")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
flag.Set("serving_dir", dir)
flag.Set("local_mirror", "../../testdata/tinymirror")
if err := logic(); err != nil {
t.Fatal(err)
}
}
================================================
FILE: cmd/debiman/mtime.go
================================================
//go:build !linux
// +build !linux
package main
import "time"
func maybeSetLinkMtime(destPath string, t time.Time) error {
return nil
}
================================================
FILE: cmd/debiman/mtime_linux.go
================================================
//go:build linux
// +build linux
package main
import (
"os"
"path/filepath"
"time"
"golang.org/x/sys/unix"
)
func maybeSetLinkMtime(destPath string, t time.Time) error {
ts := unix.NsecToTimespec(t.UnixNano())
dir, err := os.Open(filepath.Dir(destPath))
if err != nil {
return err
}
defer dir.Close()
return unix.UtimesNanoAt(int(dir.Fd()), destPath, []unix.Timespec{ts, ts}, unix.AT_SYMLINK_NOFOLLOW)
}
================================================
FILE: cmd/debiman/prometheus.go
================================================
package main
import (
"html/template"
"io"
"time"
)
const metricsTmplContent = `
# HELP packages_total The total number of Debian binary packages processed.
# TYPE packages_total gauge
packages_total {{ .Packages }}
# HELP packages_extracted Number of Debian binary packages from which manpages were extracted.
# TYPE packages_extracted gauge
packages_extracted {{ .Stats.PackagesExtracted }}
# HELP packages_deleted Number of Debian binary packages deleted because they were no longer present.
# TYPE packages_deleted gauge
packages_deleted {{ .Stats.PackagesDeleted }}
# HELP manpages_rendered Number of manpages rendered to HTML
# TYPE manpages_rendered gauge
manpages_rendered {{ .Stats.ManpagesRendered }}
# HELP manpage_bytes Total number of bytes used by manpages (by format).
# TYPE manpage_bytes gauge
manpage_bytes{format="man"} {{ .Stats.ManpageBytes }}
manpage_bytes{format="html"} {{ .Stats.HTMLBytes }}
# HELP index_bytes Total number of bytes used for the auxserver index.
# TYPE index_bytes gauge
index_bytes {{ .Stats.IndexBytes }}
# HELP runtime Wall-clock runtime in seconds.
# TYPE runtime gauge
runtime {{ .Seconds }}
# HELP last_successful_run Last successful run in seconds since the epoch.
# TYPE last_successful_run gauge
last_successful_run {{ .LastSuccessfulRun }}
`
var metricsTmpl = template.Must(template.New("metrics").Parse(metricsTmplContent))
func writeMetrics(w io.Writer, gv globalView, start time.Time) error {
now := time.Now()
return metricsTmpl.Execute(w, struct {
Packages int
Stats *stats
Now time.Time
Seconds int
LastSuccessfulRun int64
}{
Packages: len(gv.pkgs),
Stats: gv.stats,
Now: now,
Seconds: int(now.Sub(start).Seconds()),
LastSuccessfulRun: now.Unix(),
})
}
================================================
FILE: cmd/debiman/render.go
================================================
package main
import (
"compress/gzip"
"encoding/json"
"flag"
"fmt"
"html/template"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/Debian/debiman/internal/commontmpl"
"github.com/Debian/debiman/internal/convert"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/sitemap"
"github.com/Debian/debiman/internal/write"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
var (
manwalkConcurrency = flag.Int("concurrency_manwalk",
1000, // below the default 1024 open file descriptor limit
"Concurrency level for walking through binary package man directories (ulimit -n must be higher!)")
renderConcurrency = flag.Int("concurrency_render",
5,
"Concurrency level for rendering manpages using mandoc")
gzipLevel = flag.Int("gzip",
9,
"gzip compression level to use for compressing HTML versions of manpages. defaults to 9 to keep network traffic minimal, but useful to reduce for development/disaster recovery (level 1 results in a 2x speedup!)")
baseURL = flag.String("base_url",
"https://manpages.debian.org",
"Base URL (without trailing slash) to the site. Used where absolute URLs are required, e.g. sitemaps.")
)
type breadcrumb struct {
Link string
Text string
}
type breadcrumbs []breadcrumb
func (b breadcrumbs) ToJSON() template.HTML {
type item struct {
Type string `json:"@type"`
ID string `json:"@id"`
Name string `json:"name"`
}
type listItem struct {
Type string `json:"@type"`
Position int `json:"position"`
Item item `json:"item"`
}
type breadcrumbList struct {
Context string `json:"@context"`
Type string `json:"@type"`
Elements []listItem `json:"itemListElement"`
}
l := breadcrumbList{
Context: "http://schema.org",
Type: "BreadcrumbList",
Elements: make([]listItem, len(b)),
}
for idx, br := range b {
l.Elements[idx] = listItem{
Type: "ListItem",
Position: idx + 1,
Item: item{
Type: "Thing",
ID: br.Link,
Name: br.Text,
},
}
}
jsonb, err := json.Marshal(l)
if err != nil {
log.Fatal(err)
}
return template.HTML(jsonb)
}
var commonTmpls = commontmpl.MustParseCommonTmpls()
type renderingMode int
const (
regularFiles renderingMode = iota
symlinks
)
// listManpages lists all files in dir (non-recursively) and returns a map from
// filename (within dir) to *manpage.Meta.
func listManpages(dir string) (map[string]*manpage.Meta, error) {
manpageByName := make(map[string]*manpage.Meta)
files, err := os.Open(dir)
if err != nil {
return nil, err
}
defer files.Close()
var predictedEOF bool
for {
if predictedEOF {
break
}
names, err := files.Readdirnames(2048)
if err != nil {
if err == io.EOF {
break
} else {
// We avoid an additional stat syscalls for each
// binary package directory by just optimistically
// calling readdir and handling the ENOTDIR error.
if sce, ok := err.(*os.SyscallError); ok && sce.Err == syscall.ENOTDIR {
return nil, nil
}
return nil, err
}
}
// When len(names) < 2048 the next Readdirnames() call will
// result in io.EOF and can be skipped to reduce getdents(2)
// syscalls by half.
predictedEOF = len(names) < 2048
for _, fn := range names {
if !strings.HasSuffix(fn, ".gz") ||
strings.HasSuffix(fn, ".html.gz") {
continue
}
full := filepath.Join(dir, fn)
m, err := manpage.FromServingPath(*servingDir, full)
if err != nil {
// If we run into this case, our code cannot correctly
// interpret the result of ServingPath().
log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
continue
}
manpageByName[fn] = m
}
}
return manpageByName, nil
}
func renderDirectoryIndex(dir string, newestModTime time.Time) error {
st, err := os.Stat(filepath.Join(dir, "index.html.gz"))
if !*forceRerender && err == nil && st.ModTime().After(newestModTime) {
return nil
}
manpageByName, err := listManpages(dir)
if err != nil {
return err
}
if len(manpageByName) == 0 {
log.Printf("WARNING: empty directory %q, not generating package index", dir)
return nil
}
return renderPkgindex(filepath.Join(dir, "index.html.gz"), manpageByName)
}
// walkManContents walks over all entries in dir and, depending on mode, does:
// 1. send a renderJob for each regular file
// 2. send a renderJob for each symlink
func walkManContents(ctx context.Context, renderChan chan<- renderJob, dir string, mode renderingMode, gv globalView, newestModTime time.Time) (time.Time, error) {
// the invariant is: each file ending in .gz must have a corresponding .html.gz file
// the .html.gz must have a modtime that is >= the modtime of the .gz file
files, err := os.Open(dir)
if err != nil {
return newestModTime, err
}
defer files.Close()
var predictedEOF bool
for {
if predictedEOF {
break
}
names, err := files.Readdirnames(2048)
if err != nil {
if err == io.EOF {
break
} else {
// We avoid an additional stat syscalls for each
// binary package directory by just optimistically
// calling readdir and handling the ENOTDIR error.
if sce, ok := err.(*os.SyscallError); ok && sce.Err == syscall.ENOTDIR {
return newestModTime, nil
}
return newestModTime, err
}
}
// When len(names) < 2048 the next Readdirnames() call will
// result in io.EOF and can be skipped to reduce getdents(2)
// syscalls by half.
predictedEOF = len(names) < 2048
for _, fn := range names {
if !strings.HasSuffix(fn, ".gz") ||
strings.HasSuffix(fn, ".html.gz") {
continue
}
full := filepath.Join(dir, fn)
st, err := os.Lstat(full)
if err != nil {
continue
}
if st.ModTime().After(newestModTime) {
newestModTime = st.ModTime()
}
symlink := st.Mode()&os.ModeSymlink != 0
if !symlink {
atomic.AddUint64(&gv.stats.ManpageBytes, uint64(st.Size()))
}
if mode == regularFiles && symlink ||
mode == symlinks && !symlink {
continue
}
n := strings.TrimSuffix(fn, ".gz") + ".html.gz"
htmlst, err := os.Stat(filepath.Join(dir, n))
if err == nil {
atomic.AddUint64(&gv.stats.HTMLBytes, uint64(htmlst.Size()))
}
if err != nil || *forceRerender || htmlst.ModTime().Before(st.ModTime()) {
m, err := manpage.FromServingPath(*servingDir, full)
if err != nil {
// If we run into this case, our code cannot correctly
// interpret the result of ServingPath().
log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
continue
}
versions := gv.xref[m.Name]
// Replace m with its corresponding entry in versions
// so that rendermanpage() can use pointer equality to
// efficiently skip entries.
for _, v := range versions {
if v.ServingPath() == m.ServingPath() {
m = v
break
}
}
// Render dependent manpages first to properly resume
// in case debiman is interrupted.
for _, v := range versions {
if v == m || *forceRerender {
continue
}
vfull := filepath.Join(*servingDir, v.RawPath())
vfn := filepath.Join(*servingDir, v.ServingPath()+".html.gz")
vhtmlst, err := os.Stat(vfn)
if err == nil && vhtmlst.ModTime().After(gv.start) {
// The variant was already re-rendered with this globalView.
continue
}
vst, err := os.Stat(vfull)
if err != nil {
log.Printf("WARNING: stat %q: %v", vfull, err)
continue
}
vreuse := ""
if vhtmlst != nil && vhtmlst.ModTime().After(vst.ModTime()) {
vreuse = vfn
}
log.Printf("%s invalidated by %s", vfn, full)
select {
case renderChan <- renderJob{
dest: vfn,
src: vfull,
meta: v,
versions: versions,
xref: gv.xref,
modTime: vst.ModTime(),
reuse: vreuse,
}:
case <-ctx.Done():
break
}
}
var reuse string
if symlink {
link, err := os.Readlink(full)
if err == nil {
resolved := filepath.Join(dir, link)
reuse = strings.TrimSuffix(resolved, ".gz") + ".html.gz"
}
}
select {
case renderChan <- renderJob{
dest: filepath.Join(dir, n),
src: full,
meta: m,
versions: versions,
xref: gv.xref,
modTime: st.ModTime(),
reuse: reuse,
}:
case <-ctx.Done():
break
}
}
}
}
return newestModTime, nil
}
func walkContents(ctx context.Context, renderChan chan<- renderJob, whitelist map[string]bool, gv globalView) error {
sitemaps := make(map[string]time.Time)
suitedirs, err := ioutil.ReadDir(*servingDir)
if err != nil {
return err
}
for _, sfi := range suitedirs {
if !sfi.IsDir() {
continue
}
if !gv.suites[sfi.Name()] {
continue
}
bins, err := os.Open(filepath.Join(*servingDir, sfi.Name()))
if err != nil {
return err
}
defer bins.Close()
// 20000 is the order of magnitude of binary packages
// (containing manpages) in any given Debian suite, so that is
// a good value to start with.
sitemapEntries := make(map[string]time.Time, 20000)
var sitemapEntriesMu sync.RWMutex
for {
names, err := bins.Readdirnames(*manwalkConcurrency)
if err != nil {
if err == io.EOF {
break
} else {
return err
}
}
var wg errgroup.Group
for _, bfn := range names {
if whitelist != nil && !whitelist[bfn] {
continue
}
if bfn == "sourcesWithManpages.txt.gz" ||
bfn == "index.html.gz" ||
bfn == "sitemap.xml.gz" ||
bfn == ".nobackup" {
continue
}
bfn := bfn // copy
dir := filepath.Join(*servingDir, sfi.Name(), bfn)
wg.Go(func() error {
// Iterating through the same directory in all
// modes increases the chance for the dirents to
// still be cached. This is important for machines
// like manziarly.debian.org, which do not have
// enough RAM to keep all dirents cached over the
// runtime of this code path.
var newestModTime time.Time
var err error
// Render all regular files first
newestModTime, err = walkManContents(ctx, renderChan, dir, regularFiles, gv, newestModTime)
if err != nil {
return err
}
// then render all symlinks, re-using the rendered fragments
newestModTime, err = walkManContents(ctx, renderChan, dir, symlinks, gv, newestModTime)
if err != nil {
return err
}
// and finally render the package index files which need to
// consider both regular files and symlinks.
if err := renderDirectoryIndex(dir, newestModTime); err != nil {
return err
}
if !newestModTime.IsZero() {
sitemapEntriesMu.Lock()
defer sitemapEntriesMu.Unlock()
sitemapEntries[bfn] = newestModTime
}
return nil
})
}
if err := wg.Wait(); err != nil {
return err
}
}
bins.Close()
sitemapPath := filepath.Join(*servingDir, sfi.Name(), "sitemap.xml.gz")
if err := write.Atomically(sitemapPath, true, func(w io.Writer) error {
return sitemap.WriteTo(w, *baseURL+"/"+sfi.Name(), sitemapEntries)
}); err != nil {
return err
}
st, err := os.Stat(sitemapPath)
if err == nil {
sitemaps[sfi.Name()] = st.ModTime()
}
}
return write.Atomically(filepath.Join(*servingDir, "sitemapindex.xml.gz"), true, func(w io.Writer) error {
return sitemap.WriteIndexTo(w, *baseURL, sitemaps)
})
}
func writeSourceIndex(gv globalView, newestForSource map[string]time.Time) error {
// Partition by suite for reduced memory usage and better locality of file
// system access
for suite := range gv.suites {
binariesBySource := make(map[string][]string)
for _, p := range gv.pkgs {
if p.suite != suite {
continue
}
binariesBySource[p.source] = append(binariesBySource[p.source], p.binarypkg)
}
for src, binaries := range binariesBySource {
srcDir := filepath.Join(*servingDir, suite, "src:"+src)
// skip if current index file is more recent than newestForSource
st, err := os.Stat(filepath.Join(srcDir, "index.html.gz"))
if !*forceRerender && err == nil && st.ModTime().After(newestForSource[src]) {
continue
}
// Aggregate manpages of all binary packages for this source package
manpages := make(map[string]*manpage.Meta)
for _, binary := range binaries {
m, err := listManpages(filepath.Join(*servingDir, suite, binary))
if err != nil {
if os.IsNotExist(err) {
continue // The package might not contain any manpages.
}
return err
}
for k, v := range m {
manpages[k] = v
}
}
if len(manpages) == 0 {
continue // The entire source package does not contain any manpages.
}
if err := os.MkdirAll(srcDir, 0755); err != nil {
return err
}
if err := renderSrcPkgindex(filepath.Join(srcDir, "index.html.gz"), src, manpages); err != nil {
return err
}
}
}
return nil
}
func writeSourcesWithManpages(gv globalView) error {
for suite := range gv.suites {
hasManpages := make(map[string]bool)
for _, p := range gv.pkgs {
if p.suite != suite {
continue
}
hasManpages[p.source] = true
}
sourcesWithManpages := make([]string, 0, len(hasManpages))
for source := range hasManpages {
sourcesWithManpages = append(sourcesWithManpages, source)
}
sort.Strings(sourcesWithManpages)
dest := filepath.Join(*servingDir, suite, "sourcesWithManpages.txt.gz")
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
return err
}
if err := write.Atomically(dest, true, func(w io.Writer) error {
for _, source := range sourcesWithManpages {
if _, err := fmt.Fprintln(w, source); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
}
return nil
}
func renderAll(gv globalView) error {
log.Printf("Preparing inverted maps")
sourceByBinary := make(map[string]string, len(gv.pkgs))
newestForSource := make(map[string]time.Time)
for _, p := range gv.pkgs {
sourceByBinary[p.suite+"/"+p.binarypkg] = p.source
newestForSource[p.source] = time.Time{}
}
log.Printf("%d sourceByBinary entries, %d newestForSource entries", len(sourceByBinary), len(newestForSource))
eg, ctx := errgroup.WithContext(context.Background())
renderChan := make(chan renderJob)
for i := 0; i < *renderConcurrency; i++ {
eg.Go(func() error {
converter, err := convert.NewProcess()
if err != nil {
return err
}
defer converter.Kill()
// NOTE(stapelberg): gzip’s decompression phase takes the same
// time, regardless of compression level. Hence, we invest the
// maximum CPU time once to achieve the best compression.
gzipw, err := gzip.NewWriterLevel(nil, *gzipLevel)
if err != nil {
return err
}
for r := range renderChan {
n, err := rendermanpage(gzipw, converter, r)
if err != nil {
// rendermanpage writes an error page if rendering
// failed, any returned error is severe (e.g. file
// system full) and should lead to termination.
return err
}
atomic.AddUint64(&gv.stats.HTMLBytes, n)
atomic.AddUint64(&gv.stats.ManpagesRendered, 1)
}
return nil
})
}
var whitelist map[string]bool
if *onlyRender != "" {
whitelist = make(map[string]bool)
log.Printf("Restricting rendering to the following binary packages:")
for _, e := range strings.Split(strings.TrimSpace(*onlyRender), ",") {
whitelist[e] = true
log.Printf(" %q", e)
}
log.Printf("(total: %d whitelist entries)", len(whitelist))
}
if err := walkContents(ctx, renderChan, whitelist, gv); err != nil {
return err
}
close(renderChan)
if err := eg.Wait(); err != nil {
return err
}
if err := writeSourceIndex(gv, newestForSource); err != nil {
return fmt.Errorf("writing source index: %v", err)
}
if err := writeSourcesWithManpages(gv); err != nil {
return fmt.Errorf("writing sourcesWithManpages: %v", err)
}
suitedirs, err := ioutil.ReadDir(*servingDir)
if err != nil {
return err
}
for _, sfi := range suitedirs {
if !sfi.IsDir() {
continue
}
if !gv.suites[sfi.Name()] {
continue
}
bins, err := os.Open(filepath.Join(*servingDir, sfi.Name()))
if err != nil {
return err
}
defer bins.Close()
names, err := bins.Readdirnames(-1)
if err != nil {
return err
}
if err := renderContents(filepath.Join(*servingDir, fmt.Sprintf("contents-%s.html.gz", sfi.Name())), sfi.Name(), names); err != nil {
return err
}
bins.Close()
}
return nil
}
================================================
FILE: cmd/debiman/render_test.go
================================================
package main
import (
"bytes"
"fmt"
"strings"
"testing"
"github.com/Debian/debiman/internal/manpage"
)
func TestBreadcrumbsToJSON(t *testing.T) {
const breadcrumbsJSON = `{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@type":"Thing","@id":"/contents-jessie.html","name":"jessie"}},{"@type":"ListItem","position":2,"item":{"@type":"Thing","@id":"/jessie/i3-wm/index.html","name":"i3-wm"}},{"@type":"ListItem","position":3,"item":{"@type":"Thing","@id":"","name":"i3(1)"}}]}`
const Suite = "jessie"
const Binarypkg = "i3-wm"
b := breadcrumbs{
{fmt.Sprintf("/contents-%s.html", Suite), Suite},
{fmt.Sprintf("/%s/%s/index.html", Suite, Binarypkg), Binarypkg},
{"", "i3(1)"},
}
if got, want := string(b.ToJSON()), breadcrumbsJSON; got != want {
fmt.Printf("%s\n", got)
t.Fatalf("unexpected breadcrumbs JSON: got %q, want %q", got, want)
}
}
// Ensure that section names containing unsafe characters like colons
// are properly handled (and do not result in ZgotmplZ values) on pages
// like https://manpages.debian.org/trixie/foot/foot.ini.5.en.html
func TestFragmentLinkWithColon(t *testing.T) {
var buf bytes.Buffer
err := manpageTmpl.ExecuteTemplate(&buf, "manpage", manpagePrepData{
Meta: &manpage.Meta{
Name: "test",
Section: "1",
Package: &manpage.PkgMeta{
Suite: "testing",
Binarypkg: "test",
},
},
TOC: []string{"SECTION: main"},
})
if err != nil {
t.Fatal(err)
}
if strings.Contains(buf.String(), "ZgotmplZ") {
t.Fatal("ZgotmplZ in output")
}
}
================================================
FILE: cmd/debiman/renderaux.go
================================================
package main
import (
"fmt"
"html/template"
"io"
"path/filepath"
"sort"
"strings"
"github.com/Debian/debiman/internal/bundled"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/write"
)
var indexTmpl = mustParseIndexTmpl()
func mustParseIndexTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("index").Parse(bundled.Asset("index.tmpl")))
}
var faqTmpl = mustParseFaqTmpl()
func mustParseFaqTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("faq").Parse(bundled.Asset("faq.tmpl")))
}
var aboutTmpl = mustParseAboutTmpl()
func mustParseAboutTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("about").Parse(bundled.Asset("about.tmpl")))
}
type bySuiteStr []string
func (p bySuiteStr) Len() int { return len(p) }
func (p bySuiteStr) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p bySuiteStr) Less(i, j int) bool {
orderi, oki := sortOrder[p[i]]
orderj, okj := sortOrder[p[j]]
if !oki || !okj {
panic(fmt.Sprintf("either %q or %q is an unknown suite. known: %+v", p[i], p[j], sortOrder))
}
return orderi < orderj
}
func renderAux(destDir string, gv globalView) error {
suites := make([]string, 0, len(gv.suites))
for suite := range gv.suites {
suites = append(suites, suite)
}
sort.Stable(bySuiteStr(suites))
if err := write.Atomically(filepath.Join(destDir, "index.html.gz"), true, func(w io.Writer) error {
return indexTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
Suites []string
Meta *manpage.Meta
HrefLangs []*manpage.Meta
}{
Title: "index",
Suites: suites,
DebimanVersion: debimanVersion,
})
}); err != nil {
return err
}
if err := write.Atomically(filepath.Join(destDir, "faq.html.gz"), true, func(w io.Writer) error {
return faqTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
Meta *manpage.Meta
HrefLangs []*manpage.Meta
}{
Title: "FAQ",
DebimanVersion: debimanVersion,
})
}); err != nil {
return err
}
if err := write.Atomically(filepath.Join(destDir, "about.html.gz"), true, func(w io.Writer) error {
return aboutTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
Meta *manpage.Meta
HrefLangs []*manpage.Meta
}{
Title: "About",
DebimanVersion: debimanVersion,
})
}); err != nil {
return err
}
for name, content := range bundled.AssetsFiltered(func(fn string) bool {
return !strings.HasSuffix(fn, ".tmpl") && !strings.HasSuffix(fn, "style.css")
}) {
if err := write.Atomically(filepath.Join(destDir, filepath.Base(name)+".gz"), true, func(w io.Writer) error {
_, err := io.WriteString(w, content)
return err
}); err != nil {
return err
}
if err := write.Atomically(filepath.Join(destDir, filepath.Base(name)), false, func(w io.Writer) error {
_, err := io.WriteString(w, content)
return err
}); err != nil {
return err
}
}
return nil
}
================================================
FILE: cmd/debiman/rendercontents.go
================================================
package main
import (
"fmt"
"html/template"
"io"
"os"
"path/filepath"
"sort"
"github.com/Debian/debiman/internal/bundled"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/write"
)
var contentsTmpl = mustParseContentsTmpl()
func mustParseContentsTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("contents").Parse(bundled.Asset("contents.tmpl")))
}
func renderContents(dest, suite string, bins []string) error {
sort.Strings(bins)
if err := write.Atomically(dest, true, func(w io.Writer) error {
return contentsTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
Bins []string
Suite string
Meta *manpage.Meta
HrefLangs []*manpage.Meta
}{
Title: fmt.Sprintf("Contents of Debian %s", suite),
DebimanVersion: debimanVersion,
Breadcrumbs: breadcrumbs{
{fmt.Sprintf("/contents-%s.html", suite), suite},
{"", "Contents"},
},
Bins: bins,
Suite: suite,
})
}); err != nil {
return err
}
destPath := filepath.Join(*servingDir, suite, "index.html.gz")
link := fmt.Sprintf("../contents-%s.html.gz", suite)
if err := os.Symlink(link, destPath); err != nil && !os.IsExist(err) {
return err
}
return nil
}
================================================
FILE: cmd/debiman/rendermanpage.go
================================================
package main
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"html/template"
"io"
"log"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/Debian/debiman/internal/bundled"
"github.com/Debian/debiman/internal/commontmpl"
"github.com/Debian/debiman/internal/convert"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/write"
"golang.org/x/text/language"
)
const iso8601Format = "2006-01-02T15:04:05Z"
// TODO(later): move this list to a package within pault.ag/debian/?
var releaseList = []string{
"buzz",
"rex",
"bo",
"hamm",
"slink",
"potato",
"woody",
"sarge",
"etch",
"lenny",
"squeeze",
"wheezy",
"wheezy-backports",
"jessie",
"jessie-backports",
"stretch",
"stretch-backports",
"buster",
"buster-backports",
"bullseye",
"bullseye-backports",
"bookworm",
"bookworm-backports",
"trixie",
"trixie-backports",
"forky",
"forky-backports",
}
var sortOrder = make(map[string]int)
func init() {
for idx, r := range releaseList {
sortOrder[r] = idx
}
sortOrder["testing"] = sortOrder["forky"]
sortOrder["unstable"] = len(releaseList)
sortOrder["experimental"] = sortOrder["unstable"] + 1
}
// stapelberg came up with the following abbreviations:
var shortSections = map[string]string{
"1": "progs",
"2": "syscalls",
"3": "libfuncs",
"4": "files",
"5": "formats",
"6": "games",
"7": "misc",
"8": "sysadmin",
"9": "kernel",
}
// taken from man(1)
var longSections = map[string]string{
"1": "Executable programs or shell commands",
"2": "System calls (functions provided by the kernel)",
"3": "Library calls (functions within program libraries)",
"4": "Special files (usually found in /dev)",
"5": "File formats and conventions eg /etc/passwd",
"6": "Games",
"7": "Miscellaneous (including macro packages and conventions), e.g. man(7), groff(7)",
"8": "System administration commands (usually only for root)",
"9": "Kernel routines [Non standard]",
}
var manpageTmpl = mustParseManpageTmpl()
func mustParseManpageTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("manpage").
Funcs(map[string]interface{}{
"ShortSection": func(section string) string {
return shortSections[section]
},
"LongSection": func(section string) string {
return longSections[section]
},
"FragmentLink": func(fragment string) template.URL {
u := url.URL{Fragment: strings.Replace(fragment, " ", "_", -1)}
return template.URL(u.String())
},
}).
Parse(bundled.Asset("manpage.tmpl")))
}
var manpageerrorTmpl = mustParseManpageerrorTmpl()
func mustParseManpageerrorTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("manpage-error").
Funcs(map[string]interface{}{
"ShortSection": func(section string) string {
return shortSections[section]
},
"LongSection": func(section string) string {
return longSections[section]
},
"FragmentLink": func(fragment string) string {
u := url.URL{Fragment: strings.Replace(fragment, " ", "_", -1)}
return u.String()
},
}).
Parse(bundled.Asset("manpageerror.tmpl")))
}
var manpagefooterextraTmpl = mustParseManpagefooterextraTmpl()
func mustParseManpagefooterextraTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("manpage-footerextra").
Funcs(map[string]interface{}{
"Iso8601": func(t time.Time) string {
return t.UTC().Format(iso8601Format)
},
}).
Parse(bundled.Asset("manpagefooterextra.tmpl")))
}
func convertFile(converter *convert.Process, src string, resolve func(ref string) string) (doc string, toc []string, err error) {
f, err := os.Open(src)
if err != nil {
return "", nil, err
}
defer f.Close()
r, err := gzip.NewReader(f)
if err != nil {
if err == io.EOF {
// TODO: better representation of an empty manpage
return "This space intentionally left blank.", nil, nil
}
return "", nil, err
}
defer r.Close()
out, toc, err := converter.ToHTML(r, resolve)
if err != nil {
return "", nil, fmt.Errorf("convert(%q): %v", src, err)
}
return out, toc, nil
}
type byPkgAndLanguage struct {
opts []*manpage.Meta
currentpkg string
}
func (p byPkgAndLanguage) Len() int { return len(p.opts) }
func (p byPkgAndLanguage) Swap(i, j int) { p.opts[i], p.opts[j] = p.opts[j], p.opts[i] }
func (p byPkgAndLanguage) Less(i, j int) bool {
// prefer manpages from the same package
if p.opts[i].Package.Binarypkg != p.opts[j].Package.Binarypkg {
if p.opts[i].Package.Binarypkg == p.currentpkg {
return true
}
}
return p.opts[i].Language < p.opts[j].Language
}
// bestLanguageMatch returns the best manpage out of options (coming
// from current) based on text/language’s matching.
func bestLanguageMatch(current *manpage.Meta, options []*manpage.Meta) *manpage.Meta {
sort.Stable(byPkgAndLanguage{options, current.Package.Binarypkg})
if options[0].Language != "en" {
for i := 1; i < len(options); i++ {
if options[i].Language == "en" {
options = append([]*manpage.Meta{options[i]}, options...)
break
}
}
}
tags := make([]language.Tag, len(options))
for idx, m := range options {
tags[idx] = m.LanguageTag
}
// NOTE(stapelberg): it would be even better to match on the
// user’s Accept-Language HTTP header here, but that is
// incompatible with the processing model of pre-generating
// all manpages.
// TODO(stapelberg): to fix the above, we could have
// client-side javascript which queries the redirector and
// improves cross-references.
matcher := language.NewMatcher(tags)
_, idx, _ := matcher.Match(current.LanguageTag)
return options[idx]
}
type byLanguage []*manpage.Meta
func (p byLanguage) Len() int { return len(p) }
func (p byLanguage) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byLanguage) Less(i, j int) bool { return p[i].Language < p[j].Language }
type renderJob struct {
dest string
src string
meta *manpage.Meta
versions []*manpage.Meta
xref map[string][]*manpage.Meta
modTime time.Time
reuse string
}
var notYetRenderedSentinel = errors.New("Not yet rendered")
type manpagePrepData struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra template.HTML
Suites []*manpage.Meta
Versions []*manpage.Meta
Sections []*manpage.Meta
Bins []*manpage.Meta
Langs []*manpage.Meta
HrefLangs []*manpage.Meta
Meta *manpage.Meta
TOC []string
Ambiguous map[*manpage.Meta]bool
Content template.HTML
Error error
}
type bySuite []*manpage.Meta
func (p bySuite) Len() int { return len(p) }
func (p bySuite) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p bySuite) Less(i, j int) bool {
orderi, oki := sortOrder[p[i].Package.Suite]
orderj, okj := sortOrder[p[j].Package.Suite]
if !oki || !okj {
panic(fmt.Sprintf("either %q or %q is an unknown suite. known: %+v", p[i].Package.Suite, p[j].Package.Suite, sortOrder))
}
return orderi < orderj
}
type byMainSection []*manpage.Meta
func (p byMainSection) Len() int { return len(p) }
func (p byMainSection) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byMainSection) Less(i, j int) bool { return p[i].MainSection() < p[j].MainSection() }
type byBinarypkg []*manpage.Meta
func (p byBinarypkg) Len() int { return len(p) }
func (p byBinarypkg) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byBinarypkg) Less(i, j int) bool { return p[i].Package.Binarypkg < p[j].Package.Binarypkg }
func rendermanpageprep(converter *convert.Process, job renderJob) (*template.Template, manpagePrepData, error) {
meta := job.meta // for convenience
// TODO(issue): document fundamental limitation: “other languages” is imprecise: e.g. crontab(1) — are the languages for package:systemd-cron or for package:cron?
// TODO(later): to boost confidence in detecting cross-references, can we add to testdata the entire list of man page names from debian to have a good test?
// TODO(later): add plain-text version
var (
content string
toc []string
renderErr = notYetRenderedSentinel
)
if job.reuse != "" {
content, toc, renderErr = reuse(job.reuse)
if renderErr != nil {
log.Printf("WARNING: re-using %q failed: %v", job.reuse, renderErr)
}
}
if renderErr != nil {
content, toc, renderErr = convertFile(converter, job.src, func(ref string) string {
idx := strings.LastIndex(ref, "(")
if idx == -1 {
return ""
}
section := ref[idx+1 : len(ref)-1]
name := ref[:idx]
related, ok := job.xref[name]
if !ok {
return ""
}
filtered := make([]*manpage.Meta, 0, len(related))
for _, r := range related {
if r.MainSection() != section {
continue
}
if r.Package.Suite != meta.Package.Suite {
continue
}
filtered = append(filtered, r)
}
if len(filtered) == 0 {
return ""
}
return commontmpl.BaseURLPath() + "/" + bestLanguageMatch(meta, filtered).ServingPath() + ".html"
})
}
log.Printf("rendering %q", job.dest)
suites := make([]*manpage.Meta, 0, len(job.versions))
for _, v := range job.versions {
if !v.Package.SameBinary(meta.Package) {
continue
}
if v.Section != meta.Section {
continue
}
// TODO(later): allow switching to a different suite even if
// switching requires a language-change. we should indicate
// this in the UI.
if v.Language != meta.Language {
continue
}
suites = append(suites, v)
}
sort.Stable(bySuite(suites))
bySection := make(map[string][]*manpage.Meta)
for _, v := range job.versions {
if v.Package.Suite != meta.Package.Suite {
continue
}
bySection[v.Section] = append(bySection[v.Section], v)
}
sections := make([]*manpage.Meta, 0, len(bySection))
for _, all := range bySection {
sections = append(sections, bestLanguageMatch(meta, all))
}
sort.Stable(byMainSection(sections))
conflicting := make(map[string]bool)
bins := make([]*manpage.Meta, 0, len(job.versions))
for _, v := range job.versions {
if v.Section != meta.Section {
continue
}
if v.Package.Suite != meta.Package.Suite {
continue
}
// We require a strict match for the language when determining
// conflicting packages, because otherwise the packages might
// be augmenting, not conflicting: crontab(1) is present in
// cron, but its translations are shipped e.g. in
// manpages-fr-extra.
if v.Language != meta.Language {
continue
}
if v.Package.Binarypkg != meta.Package.Binarypkg {
conflicting[v.Package.Binarypkg] = true
}
bins = append(bins, v)
}
sort.Stable(byBinarypkg(bins))
ambiguous := make(map[*manpage.Meta]bool)
byLang := make(map[string][]*manpage.Meta)
for _, v := range job.versions {
if v.Section != meta.Section {
continue
}
if v.Package.Suite != meta.Package.Suite {
continue
}
if conflicting[v.Package.Binarypkg] {
continue
}
byLang[v.Language] = append(byLang[v.Language], v)
}
langs := make([]*manpage.Meta, 0, len(byLang))
hrefLangs := make([]*manpage.Meta, 0, len(byLang))
for _, all := range byLang {
for _, e := range all {
langs = append(langs, e)
if len(all) > 1 {
ambiguous[e] = true
}
// hreflang consists only of language and region,
// scripts are not supported.
if !strings.Contains(e.Language, "@") {
hrefLangs = append(hrefLangs, e)
}
}
}
// Sort alphabetically by the locale names (e.g. zh_TW).
sort.Sort(byLanguage(langs))
sort.Sort(byLanguage(hrefLangs))
t := manpageTmpl
title := fmt.Sprintf("%s(%s) — %s — Debian %s", meta.Name, meta.Section, meta.Package.Binarypkg, meta.Package.Suite)
shorttitle := fmt.Sprintf("%s(%s)", meta.Name, meta.Section)
if renderErr != nil {
t = manpageerrorTmpl
title = "Error: " + title
}
var footerExtra bytes.Buffer
if err := manpagefooterextraTmpl.Execute(&footerExtra, struct {
SourceFile string
LastUpdated time.Time
Converted time.Time
Meta *manpage.Meta
}{
SourceFile: filepath.Base(job.src),
LastUpdated: job.modTime,
Converted: time.Now(),
Meta: meta,
}); err != nil {
return nil, manpagePrepData{}, err
}
return t, manpagePrepData{
Title: title,
DebimanVersion: debimanVersion,
Breadcrumbs: breadcrumbs{
{fmt.Sprintf("/contents-%s.html", meta.Package.Suite), meta.Package.Suite},
{fmt.Sprintf("/%s/%s/index.html", meta.Package.Suite, meta.Package.Binarypkg), meta.Package.Binarypkg},
{"", shorttitle},
},
FooterExtra: template.HTML(footerExtra.String()),
Suites: suites,
Versions: job.versions,
Sections: sections,
Bins: bins,
Langs: langs,
HrefLangs: hrefLangs,
Meta: meta,
TOC: toc,
Ambiguous: ambiguous,
Content: template.HTML(content),
Error: renderErr,
}, nil
}
type countingWriter int64
func (c *countingWriter) Write(p []byte) (n int, err error) {
*c += countingWriter(len(p))
return len(p), nil
}
func rendermanpage(gzipw *gzip.Writer, converter *convert.Process, job renderJob) (uint64, error) {
t, data, err := rendermanpageprep(converter, job)
if err != nil {
return 0, err
}
var written countingWriter
if err := write.AtomicallyWithGz(job.dest, gzipw, func(w io.Writer) error {
return t.Execute(io.MultiWriter(w, &written), data)
}); err != nil {
return 0, err
}
return uint64(written), nil
}
================================================
FILE: cmd/debiman/rendermanpage_test.go
================================================
package main
import (
"compress/gzip"
"io/ioutil"
"os"
"testing"
"time"
"github.com/Debian/debiman/internal/convert"
"github.com/Debian/debiman/internal/manpage"
)
func mustParseFromServingPath(t *testing.T, path string) *manpage.Meta {
m, err := manpage.FromServingPath("/srv/man", path)
if err != nil {
t.Fatal(err)
}
return m
}
func TestBestLanguageMatch(t *testing.T) {
table := []struct {
current *manpage.Meta
options []*manpage.Meta
wantServingPath string
}{
{
current: mustParseFromServingPath(t, "testing/cron/crontab.1.fr"),
options: []*manpage.Meta{
mustParseFromServingPath(t, "testing/systemd-cron/crontab.5.fr"),
mustParseFromServingPath(t, "testing/cron/crontab.5.fr"),
mustParseFromServingPath(t, "testing/cron/crontab.5.en"),
},
wantServingPath: "testing/cron/crontab.5.fr",
},
}
for _, entry := range table {
entry := entry // capture
t.Run(entry.wantServingPath, func(t *testing.T) {
t.Parallel()
best := bestLanguageMatch(entry.current, entry.options)
if got, want := best.ServingPath(), entry.wantServingPath; got != want {
t.Fatalf("Unexpected best language match: got %q, want %q", got, want)
}
})
}
}
func TestPrep(t *testing.T) {
const manContents = `.SH foobar
baz
.SH qux
`
f, err := ioutil.TempFile("", "debiman-test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
gzipw := gzip.NewWriter(f)
if _, err := gzipw.Write([]byte(manContents)); err != nil {
t.Fatal(err)
}
if err := gzipw.Close(); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
manpagesFrExtra5 := mustParseFromServingPath(t, "jessie/manpages-fr-extra/crontab.5.fr")
manpagesFrExtra1 := mustParseFromServingPath(t, "jessie/manpages-fr-extra/crontab.1.fr")
manpagesJa := mustParseFromServingPath(t, "jessie/manpages-ja/crontab.5.ja")
systemdCron := mustParseFromServingPath(t, "jessie/systemd-cron/crontab.5.en")
cron := mustParseFromServingPath(t, "jessie/cron/crontab.5.en")
bcronRun := mustParseFromServingPath(t, "jessie/bcron-run/crontab.5.en")
// Pretend crontab.5.en moved to manpages-fr-systemd for testing issue #27
manpagesFrSystemd := mustParseFromServingPath(t, "testing/manpages-fr-systemd/crontab.5.fr")
manpagesFrSystemd.Package.Replaces = []string{"manpages-fr-extra"}
converter, err := convert.NewProcess()
if err != nil {
t.Fatal(err)
}
defer converter.Kill()
_, data, err := rendermanpageprep(converter, renderJob{
dest: f.Name(),
src: f.Name(),
meta: manpagesFrExtra5,
versions: []*manpage.Meta{
manpagesFrExtra5,
manpagesFrExtra1,
manpagesJa,
systemdCron,
cron,
bcronRun,
manpagesFrSystemd,
},
xref: map[string][]*manpage.Meta{
"crontab": []*manpage.Meta{
manpagesFrExtra5,
manpagesFrExtra1,
manpagesJa,
systemdCron,
cron,
bcronRun,
manpagesFrSystemd,
},
},
modTime: time.Now(),
})
if err != nil {
t.Fatal(err)
}
t.Run("versions", func(t *testing.T) {
wantSuites := []*manpage.Meta{
manpagesFrExtra5,
manpagesFrSystemd,
}
if got, want := len(data.Suites), len(wantSuites); got != want {
t.Fatalf("unexpected number of data.Suites: got %d, want %d", got, want)
}
for i := 0; i < len(data.Suites); i++ {
if got, want := data.Suites[i], wantSuites[i]; got != want {
t.Fatalf("unexpected entry in data.Suites: got %v, want %v", got, want)
}
}
})
t.Run("lang", func(t *testing.T) {
wantLang := []*manpage.Meta{
systemdCron,
cron,
bcronRun,
manpagesFrExtra5,
manpagesJa,
}
if got, want := len(data.Langs), len(wantLang); got != want {
t.Fatalf("unexpected number of data.Langs: got %d, want %d", got, want)
}
for i := 0; i < len(data.Langs); i++ {
if got, want := data.Langs[i], wantLang[i]; got != want {
t.Fatalf("unexpected entry in data.Langs: got %v, want %v", got, want)
}
}
})
t.Run("section", func(t *testing.T) {
wantSections := []*manpage.Meta{
manpagesFrExtra1,
manpagesFrExtra5,
}
if got, want := len(data.Sections), len(wantSections); got != want {
t.Fatalf("unexpected number of data.Sections: got %d, want %d", got, want)
}
for i := 0; i < len(data.Sections); i++ {
if got, want := data.Sections[i], wantSections[i]; got != want {
t.Fatalf("unexpected entry in data.Sections: got %v, want %v", got, want)
}
}
})
t.Run("ambiguous", func(t *testing.T) {
wantAmbiguous := map[*manpage.Meta]bool{
systemdCron: true,
cron: true,
bcronRun: true,
}
if got, want := len(data.Ambiguous), len(wantAmbiguous); got != want {
t.Fatalf("unexpected number of data.Ambiguous: got %d, want %d", got, want)
}
for want := range wantAmbiguous {
if _, ok := data.Ambiguous[want]; !ok {
t.Fatalf("data.Ambiguous unexpectedly does not contain key %v", want)
}
}
})
}
================================================
FILE: cmd/debiman/renderpkgindex.go
================================================
package main
import (
"fmt"
"html/template"
"io"
"sort"
"github.com/Debian/debiman/internal/bundled"
"github.com/Debian/debiman/internal/manpage"
"github.com/Debian/debiman/internal/write"
)
var pkgindexTmpl = mustParsePkgindexTmpl()
var srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
func mustParsePkgindexTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("pkgindex").Parse(bundled.Asset("pkgindex.tmpl")))
}
func mustParseSrcPkgindexTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("srcpkgindex").Parse(bundled.Asset("srcpkgindex.tmpl")))
}
func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
var first *manpage.Meta
for _, m := range manpageByName {
first = m
break
}
mans := make([]string, 0, len(manpageByName))
for n := range manpageByName {
mans = append(mans, n)
}
sort.Strings(mans)
return write.Atomically(dest, true, func(w io.Writer) error {
return pkgindexTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
First *manpage.Meta
Meta *manpage.Meta
ManpageByName map[string]*manpage.Meta
Mans []string
HrefLangs []*manpage.Meta
}{
Title: fmt.Sprintf("Manpages of %s in Debian %s", first.Package.Binarypkg, first.Package.Suite),
DebimanVersion: debimanVersion,
Breadcrumbs: breadcrumbs{
{fmt.Sprintf("/contents-%s.html", first.Package.Suite), first.Package.Suite},
{fmt.Sprintf("/%s/%s/index.html", first.Package.Suite, first.Package.Binarypkg), first.Package.Binarypkg},
{"", "Contents"},
},
First: first,
Meta: first,
ManpageByName: manpageByName,
Mans: mans,
})
})
}
func renderSrcPkgindex(dest string, src string, manpageByName map[string]*manpage.Meta) error {
var first *manpage.Meta
for _, m := range manpageByName {
first = m
break
}
mans := make([]string, 0, len(manpageByName))
for n := range manpageByName {
mans = append(mans, n)
}
sort.Strings(mans)
return write.Atomically(dest, true, func(w io.Writer) error {
return srcpkgindexTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
First *manpage.Meta
Meta *manpage.Meta
ManpageByName map[string]*manpage.Meta
Mans []string
HrefLangs []*manpage.Meta
Src string
}{
Title: fmt.Sprintf("Manpages of src:%s in Debian %s", src, first.Package.Suite),
DebimanVersion: debimanVersion,
Breadcrumbs: breadcrumbs{
{fmt.Sprintf("/contents-%s.html", first.Package.Suite), first.Package.Suite},
{fmt.Sprintf("/%s/src:%s/index.html", first.Package.Suite, src), "src:" + src},
{"", "Contents"},
},
First: first,
Meta: first,
ManpageByName: manpageByName,
Mans: mans,
Src: src,
})
})
}
================================================
FILE: cmd/debiman/reuse.go
================================================
package main
import (
"bufio"
"bytes"
"compress/gzip"
"os"
)
var mandocDivB = []byte(``)
var tocLinkPrefix = []byte(`
\n \n"))
return string(bytes.TrimSpace(all)), toc, nil
}
if inManpage {
if _, err := buf.Write(b); err != nil {
return "", nil, err
}
if _, err := buf.Write([]byte{'\n'}); err != nil {
return "", nil, err
}
} else if bytes.HasPrefix(b, tocLinkPrefix) {
entry := bytes.TrimSuffix(b, []byte("