Browse Source

Implement documentation search

pull/699/head
Pēteris Caune 2 years ago
parent
commit
5d5e469347
No known key found for this signature in database GPG Key ID: E28D7679E9A9EDE2
  1. 1
      CHANGELOG.md
  2. 4
      hc/front/forms.py
  3. 38
      hc/front/management/commands/populate_searchdb.py
  4. 17
      hc/front/tests/test_search.py
  5. 1
      hc/front/urls.py
  6. 66
      hc/front/views.py
  7. 10
      hc/lib/html.py
  8. BIN
      search.db
  9. 23
      static/css/search.css
  10. 31
      static/js/search.js
  11. 1
      templates/base.html
  12. 61
      templates/front/base_docs.html
  13. 14
      templates/front/docs_search.html
  14. 72
      templates/front/docs_single.html

1
CHANGELOG.md

@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
### Improvements
- Add support for EMAIL_USE_SSL environment variable (#685)
- Switch from requests to pycurl
- Implement documentation search
### Bug Fixes
- Fix the handling of TooManyRedirects exceptions

4
hc/front/forms.py

@ -333,3 +333,7 @@ class AddGotifyForm(forms.Form):
def get_value(self):
return json.dumps(dict(self.cleaned_data), sort_keys=True)
class SearchForm(forms.Form):
q = forms.RegexField(regex=r"^[0-9a-zA-Z\s]{3,100}$")

38
hc/front/management/commands/populate_searchdb.py

@ -0,0 +1,38 @@
import os
import sqlite3
from django.conf import settings
from django.core.management.base import BaseCommand
from hc.front.views import _replace_placeholders
from hc.lib.html import html2text
class Command(BaseCommand):
help = "Renders Markdown to HTML"
def handle(self, *args, **options):
con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
cur = con.cursor()
cur.execute("DROP TABLE IF EXISTS docs")
cur.execute(
"""CREATE VIRTUAL TABLE docs USING FTS5(slug, title, body, tokenize="trigram")"""
)
docs_path = os.path.join(settings.BASE_DIR, "templates/docs")
for filename in os.listdir(docs_path):
if not filename.endswith(".html"):
continue
slug = filename[:-5] # cut ".html"
print("Processing %s" % slug)
html = open(os.path.join(docs_path, filename), "r").read()
html = _replace_placeholders(slug, html)
lines = html.split("\n")
title = html2text(lines[0])
text = html2text("\n".join(lines[1:]), skip_pre=True)
cur.execute("INSERT INTO docs VALUES (?, ?, ?)", (slug, title, text))
con.commit()

17
hc/front/tests/test_search.py

@ -0,0 +1,17 @@
from hc.test import BaseTestCase
class SearchTestCase(BaseTestCase):
def test_it_works(self):
r = self.client.get("/docs/search/?q=failure")
self.assertContains(
r, "You can actively signal a <span>failure</span>", status_code=200
)
def test_it_handles_no_results(self):
r = self.client.get("/docs/search/?q=asfghjkl")
self.assertContains(r, "Your search query matched no results", status_code=200)
def test_it_rejects_special_characters(self):
r = self.client.get("/docs/search/?q=api/v1")
self.assertContains(r, "Your search query matched no results", status_code=200)

1
hc/front/urls.py

@ -106,5 +106,6 @@ urlpatterns = [
path("projects/<uuid:code>/", include(project_urls)),
path("docs/", views.serve_doc, name="hc-docs"),
path("docs/cron/", views.docs_cron, name="hc-docs-cron"),
path("docs/search/", views.docs_search, name="hc-docs-search"),
path("docs/<slug:doc>/", views.serve_doc, name="hc-serve-doc"),
]

66
hc/front/views.py

@ -5,6 +5,7 @@ import json
import os
import re
from secrets import token_urlsafe
import sqlite3
from urllib.parse import urlencode, urlparse
from cron_descriptor import ExpressionDescriptor
@ -348,6 +349,29 @@ def dashboard(request):
return render(request, "front/dashboard.html", {})
def _replace_placeholders(doc, html):
if doc.startswith("self_hosted"):
return html
replaces = {
"{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
"{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
"SITE_NAME": settings.SITE_NAME,
"SITE_ROOT": settings.SITE_ROOT,
"SITE_HOSTNAME": site_hostname(),
"SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
"PING_ENDPOINT": settings.PING_ENDPOINT,
"PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
"PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
"IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
}
for placeholder, value in replaces.items():
html = html.replace(placeholder, value)
return html
def serve_doc(request, doc="introduction"):
# Filenames in /templates/docs/ consist of lowercase letters and underscores,
# -- make sure we don't accept anything else
@ -361,23 +385,7 @@ def serve_doc(request, doc="introduction"):
with open(path, "r", encoding="utf-8") as f:
content = f.read()
if not doc.startswith("self_hosted"):
replaces = {
"{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
"{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
"SITE_NAME": settings.SITE_NAME,
"SITE_ROOT": settings.SITE_ROOT,
"SITE_HOSTNAME": site_hostname(),
"SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
"PING_ENDPOINT": settings.PING_ENDPOINT,
"PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
"PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
"IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
}
for placeholder, value in replaces.items():
content = content.replace(placeholder, value)
content = _replace_placeholders(doc, content)
ctx = {
"page": "docs",
"section": doc,
@ -388,6 +396,30 @@ def serve_doc(request, doc="introduction"):
return render(request, "front/docs_single.html", ctx)
@csrf_exempt
def docs_search(request):
form = forms.SearchForm(request.GET)
if not form.is_valid():
ctx = {"results": []}
return render(request, "front/docs_search.html", ctx)
query = """
SELECT slug, title, snippet(docs, 2, '<span>', '</span>', '&hellip;', 50)
FROM docs
WHERE docs MATCH ?
ORDER BY bm25(docs, 2.0, 10.0, 1.0)
LIMIT 8
"""
q = form.cleaned_data["q"]
con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
cur = con.cursor()
res = cur.execute(query, (q,))
ctx = {"results": res.fetchall()}
return render(request, "front/docs_search.html", ctx)
def docs_cron(request):
return render(request, "front/docs_cron.html", {})

10
hc/lib/html.py

@ -6,13 +6,14 @@ class TextOnlyParser(HTMLParser):
super().__init__(*args, **kwargs)
self.active = True
self.buf = []
self.skiplist = set(["script", "style"])
def handle_starttag(self, tag, attrs):
if tag in ("script", "style"):
if tag in self.skiplist:
self.active = False
def handle_endtag(self, tag):
if tag in ("script", "style"):
if tag in self.skiplist:
self.active = True
def handle_data(self, data):
@ -24,7 +25,10 @@ class TextOnlyParser(HTMLParser):
return " ".join(messy.split())
def html2text(html):
def html2text(html, skip_pre=False):
parser = TextOnlyParser()
if skip_pre:
parser.skiplist.add("pre")
parser.feed(html)
return parser.get_text()

BIN
search.db

23
static/css/search.css

@ -0,0 +1,23 @@
@media (min-width: 992px) {
#docs-search-form {
margin-right: 20%;
}
}
#search-results {
display: none;
}
#search-results.on {
display: block;
}
.docs-nav.off {
display: none;
}
#search-results li span {
background: #ffef82;
border-radius: 2px;
color: #111;
}

31
static/js/search.js

@ -0,0 +1,31 @@
$(function() {
var base = document.getElementById("base-url").getAttribute("href").slice(0, -1);
var input = $("#docs-search");
input.on("keyup focus", function() {
var q = this.value;
if (q.length < 3) {
$("#search-results").removeClass("on");
$("#docs-nav").removeClass("off");
return
}
$.ajax({
url: base + "/docs/search/",
type: "get",
data: {q: q},
success: function(data) {
if (q != input.val()) {
return; // ignore stale results
}
$("#search-results").html(data).addClass("on");
$("#docs-nav").addClass("off");
}
});
});
// input.on("blur", function() {
// $("#search-results").removeClass("on");
// });
});

1
templates/base.html

@ -52,6 +52,7 @@
<link rel="stylesheet" href="{% static 'css/profile.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/projects.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/radio.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/search.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/settings.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/snippet-copy.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/syntax.css' %}" type="text/css">

61
templates/front/base_docs.html

@ -1,61 +0,0 @@
{% extends "base.html" %}
{% load hc_extras %}
{% block content %}
<div class="row">
<div class="col-sm-3">
<ul class="docs-nav">
<li class="nav-header">{{ site_name }}</li>
<li {% if section == "introduction" %} class="active"{% endif %}>
<a href="{% url 'hc-docs' %}">Introduction</a>
</li>
{% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
{% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
{% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
{% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
<li class="nav-header">API</li>
{% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
{% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
<li class="nav-header">Pinging Examples</li>
{% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
{% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
{% include "front/docs_nav_item.html" with slug="python" title="Python" %}
{% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
{% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
{% include "front/docs_nav_item.html" with slug="go" title="Go" %}
{% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
{% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
{% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
{% include "front/docs_nav_item.html" with slug="email" title="Email" %}
<li class="nav-header">Guides</li>
{% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
{% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
{% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
{% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
{% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
{% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
<li class="nav-header">Developer Tools</li>
{% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
<li class="nav-header">Self-hosted</li>
{% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
{% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
{% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
<li class="nav-header">Reference</li>
<li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
</ul>
</div>
<div class="col-sm-9">
{% block docs_content %}
{% endblock %}
</div>
</div>
{% endblock %}

14
templates/front/docs_search.html

@ -0,0 +1,14 @@
<ul class="docs-nav">
<li class="nav-header">Search Results</li>
{% if results %}
{% for slug, title, snippet in results %}
<li>
<a href="{% url 'hc-serve-doc' slug %}">{{ title|safe }}</a>
<p>{{ snippet|safe }}</p>
</li>
{% endfor %}
</ul>
{% else %}
<li>Your search query matched no results.</li>
{% endif %}
</ul>

72
templates/front/docs_single.html

@ -1,12 +1,75 @@
{% extends "front/base_docs.html" %}
{% load compress static hc_extras %}
{% extends "base.html" %}
{% load compress hc_extras static %}
{% block title %}{{ first_line|striptags }} - {{ site_name }}{% endblock %}
{% block description %}{% endblock %}
{% block keywords %}{% endblock %}
{% block docs_content %}
<div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
{% block content %}
<div class="row">
<div class="col-sm-3">
<div id="docs-search-form">
<input
id="docs-search"
type="text"
name="q"
class="form-control input-sm"
placeholder="Search docs&hellip;"
autocomplete="off">
</div>
<div id="search-results"></div>
<ul id="docs-nav" class="docs-nav">
<li class="nav-header">{{ site_name }}</li>
<li {% if section == "introduction" %} class="active"{% endif %}>
<a href="{% url 'hc-docs' %}">Introduction</a>
</li>
{% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
{% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
{% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
{% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
<li class="nav-header">API</li>
{% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
{% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
<li class="nav-header">Pinging Examples</li>
{% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
{% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
{% include "front/docs_nav_item.html" with slug="python" title="Python" %}
{% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
{% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
{% include "front/docs_nav_item.html" with slug="go" title="Go" %}
{% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
{% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
{% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
{% include "front/docs_nav_item.html" with slug="email" title="Email" %}
<li class="nav-header">Guides</li>
{% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
{% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
{% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
{% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
{% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
{% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
<li class="nav-header">Developer Tools</li>
{% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
<li class="nav-header">Self-hosted</li>
{% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
{% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
{% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
<li class="nav-header">Reference</li>
<li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
</ul>
</div>
<div class="col-sm-9">
<div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
</div>
</div>
{% endblock %}
{% block scripts %}
@ -15,5 +78,6 @@
<script src="{% static 'js/bootstrap.min.js' %}"></script>
<script src="{% static 'js/clipboard.min.js' %}"></script>
<script src="{% static 'js/snippet-copy.js' %}"></script>
<script src="{% static 'js/search.js' %}"></script>
{% endcompress %}
{% endblock %}
Loading…
Cancel
Save