Phase 4: upload, LLM extraction, import review flow

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Jānis Kacēns
2026-05-11 13:15:04 +03:00
parent e53e7662e9
commit 5199c1fa16
10 changed files with 447 additions and 6 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
OPENAI_API_KEY=sk-...
SESSION_SECRET=change-me-to-a-random-32-char-string
DATA_DIR=./data
PORT=8080
PORT=8079
ADMIN_USERS=alice:password1,bob:password2
LLM_MODEL=gpt-4o-mini
+9 -3
View File
@@ -22,6 +22,7 @@ import (
"qbank/internal/config"
"qbank/internal/db"
"qbank/internal/handlers"
"qbank/internal/llm"
)
func main() {
@@ -50,11 +51,13 @@ func main() {
repo := db.New(database)
authMgr := auth.NewManager(database)
renderer := handlers.NewRenderer("web/templates")
llmClient := llm.New(cfg.OpenAIAPIKey, cfg.LLMModel)
ensureIcons("web/static")
authH := handlers.NewAuthHandler(authMgr, repo, renderer)
homeH := handlers.NewHomeHandler(authMgr, renderer)
uploadH := handlers.NewUploadHandler(authMgr, repo, llmClient, renderer, cfg.DataDir)
r := chi.NewRouter()
r.Use(middleware.RequestID)
@@ -78,13 +81,17 @@ func main() {
r.Group(func(r chi.Router) {
r.Use(authMgr.RequireAuth)
r.Get("/", homeH.Handle)
r.Get("/upload", uploadH.UploadGet)
r.Post("/upload", uploadH.UploadPost)
r.Get("/import/{id}", uploadH.ImportGet)
r.Post("/import/{id}", uploadH.ImportPost)
})
srv := &http.Server{
Addr: ":" + cfg.Port,
Handler: r,
ReadTimeout: 15 * time.Second,
WriteTimeout: 30 * time.Second,
WriteTimeout: 120 * time.Second, // LLM extraction can take a while
IdleTimeout: 60 * time.Second,
}
@@ -108,7 +115,6 @@ func main() {
slog.Info("server stopped")
}
// ensureIcons generates simple solid-color PNG icons if they don't already exist.
func ensureIcons(dir string) {
for _, size := range []int{192, 512} {
path := filepath.Join(dir, fmt.Sprintf("icon-%d.png", size))
@@ -116,7 +122,7 @@ func ensureIcons(dir string) {
continue
}
img := image.NewRGBA(image.Rect(0, 0, size, size))
blue := color.RGBA{R: 37, G: 99, B: 235, A: 255} // Tailwind blue-600
blue := color.RGBA{R: 37, G: 99, B: 235, A: 255}
draw.Draw(img, img.Bounds(), &image.Uniform{C: blue}, image.Point{}, draw.Src)
f, err := os.Create(path)
if err != nil {
+1 -1
View File
@@ -24,7 +24,7 @@ func Load() *Config {
OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"),
SessionSecret: os.Getenv("SESSION_SECRET"),
DataDir: envOr("DATA_DIR", "./data"),
Port: envOr("PORT", "8080"),
Port: envOr("PORT", "8079"),
LLMModel: envOr("LLM_MODEL", "gpt-4o-mini"),
}
cfg.AdminUsers = parseAdminUsers(os.Getenv("ADMIN_USERS"))
+48
View File
@@ -1,8 +1,10 @@
package db
import (
"crypto/rand"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
@@ -373,3 +375,49 @@ func (r *Repo) GetStatsForUser(userID int64, questionIDs []string) (map[string]*
}
return result, rows.Err()
}
// ── Draft (import review) ────────────────────────────────────────────────────
func newDraftID() string {
b := make([]byte, 16)
rand.Read(b)
return hex.EncodeToString(b)
}
func (r *Repo) CreateDraft(userID int64, source string, questions []models.DraftQuestion) (string, error) {
data, err := json.Marshal(questions)
if err != nil {
return "", err
}
id := newDraftID()
_, err = r.db.Exec(
"INSERT INTO import_drafts (id, user_id, source, questions) VALUES (?, ?, ?, ?)",
id, userID, source, string(data),
)
if err != nil {
return "", err
}
return id, nil
}
func (r *Repo) GetDraftForUser(id string, userID int64) (*models.Draft, error) {
d := &models.Draft{}
var questionsJSON, createdAt string
err := r.db.QueryRow(
"SELECT id, user_id, source, questions, created_at FROM import_drafts WHERE id = ? AND user_id = ?",
id, userID,
).Scan(&d.ID, &d.UserID, &d.Source, &questionsJSON, &createdAt)
if err != nil {
return nil, err
}
d.CreatedAt = parseTime(createdAt)
if err := json.Unmarshal([]byte(questionsJSON), &d.Questions); err != nil {
return nil, err
}
return d, nil
}
func (r *Repo) DeleteDraft(id string) error {
_, err := r.db.Exec("DELETE FROM import_drafts WHERE id = ?", id)
return err
}
+8
View File
@@ -54,6 +54,14 @@ CREATE TABLE IF NOT EXISTS sessions (
);
CREATE INDEX IF NOT EXISTS idx_sessions_expiry ON sessions(expiry);
CREATE TABLE IF NOT EXISTS import_drafts (
id TEXT PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES users(id),
source TEXT NOT NULL DEFAULT '',
questions TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_test_answers_test ON test_answers(test_id);
CREATE INDEX IF NOT EXISTS idx_answers_question ON answers(question_id);
CREATE INDEX IF NOT EXISTS idx_stats_user ON user_question_stats(user_id);
+5 -1
View File
@@ -9,6 +9,10 @@ import (
"qbank/internal/auth"
)
var tmplFuncs = template.FuncMap{
"inc": func(i int) int { return i + 1 },
}
// Renderer parses and executes HTML templates from a directory.
type Renderer struct {
dir string
@@ -18,7 +22,7 @@ func NewRenderer(dir string) *Renderer { return &Renderer{dir: dir} }
// Render executes layout.html + <name>.html, passing data to the "layout" template.
func (r *Renderer) Render(w http.ResponseWriter, status int, name string, data any) {
t, err := template.ParseFiles(
t, err := template.New("").Funcs(tmplFuncs).ParseFiles(
filepath.Join(r.dir, "layout.html"),
filepath.Join(r.dir, name+".html"),
)
+233
View File
@@ -0,0 +1,233 @@
package handlers
import (
"bytes"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/go-chi/chi/v5"
"qbank/internal/auth"
"qbank/internal/db"
"qbank/internal/llm"
"qbank/internal/models"
"qbank/internal/parse"
)
type UploadHandler struct {
auth *auth.Manager
repo *db.Repo
llm *llm.Client
render *Renderer
dataDir string
}
func NewUploadHandler(a *auth.Manager, repo *db.Repo, llmClient *llm.Client, r *Renderer, dataDir string) *UploadHandler {
return &UploadHandler{auth: a, repo: repo, llm: llmClient, render: r, dataDir: dataDir}
}
func (h *UploadHandler) UploadGet(w http.ResponseWriter, r *http.Request) {
data := BaseData(h.auth, r)
h.render.Render(w, http.StatusOK, "upload", data)
}
func (h *UploadHandler) UploadPost(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 20<<20) // 20 MB
renderErr := func(msg string) {
data := BaseData(h.auth, r)
data["Error"] = msg
h.render.Render(w, http.StatusUnprocessableEntity, "upload", data)
}
if !h.auth.CheckCSRF(r) {
HTTPError(w, http.StatusForbidden)
return
}
if err := r.ParseMultipartForm(20 << 20); err != nil {
renderErr("File too large (max 20 MB).")
return
}
file, header, err := r.FormFile("file")
if err != nil {
renderErr("No file selected.")
return
}
defer file.Close()
ext := strings.ToLower(filepath.Ext(header.Filename))
if ext != ".pdf" && ext != ".docx" {
renderErr("Unsupported file type. Upload a .pdf or .docx file.")
return
}
// Read file once; use for both saving and extraction.
data, err := io.ReadAll(file)
if err != nil {
renderErr("Failed to read uploaded file.")
return
}
// Save to disk.
uploadsDir := filepath.Join(h.dataDir, "uploads")
if err := os.MkdirAll(uploadsDir, 0755); err != nil {
renderErr("Server error: could not create uploads directory.")
return
}
safeName := filepath.Base(header.Filename)
savedPath := filepath.Join(uploadsDir, fmt.Sprintf("%d_%s", time.Now().UnixMilli(), safeName))
if err := os.WriteFile(savedPath, data, 0644); err != nil {
renderErr("Server error: could not save file.")
return
}
// Extract text.
var text string
switch ext {
case ".pdf":
text, err = parse.ExtractPDF(bytes.NewReader(data))
if errors.Is(err, parse.ErrScanPDF) {
renderErr("This PDF appears to be image-only (scan-based). Please convert it to a text PDF first.")
return
}
case ".docx":
text, err = parse.ExtractDOCX(bytes.NewReader(data))
}
if err != nil {
renderErr("Could not extract text from the document: " + err.Error())
return
}
// Chunk and call LLM.
chunks := parse.Chunk(text, 10_000)
seen := make(map[string]bool)
var draftQs []models.DraftQuestion
for _, chunk := range chunks {
qs, err := h.llm.ExtractQuestions(r.Context(), chunk)
if err != nil {
slog.Warn("llm chunk failed", "err", err)
continue
}
for _, q := range qs {
key := db.QuestionID(q.Question)
if seen[key] {
continue
}
seen[key] = true
dq := models.DraftQuestion{Text: q.Question}
for _, a := range q.Answers {
dq.Answers = append(dq.Answers, models.DraftAnswer{Text: a.Text, IsCorrect: a.Correct})
}
draftQs = append(draftQs, dq)
}
}
source := strings.TrimSpace(r.FormValue("source"))
if source == "" {
source = strings.TrimSuffix(safeName, ext)
}
userID := auth.UserFromCtx(r.Context()).ID
draftID, err := h.repo.CreateDraft(userID, source, draftQs)
if err != nil {
renderErr("Server error: could not create import draft.")
return
}
http.Redirect(w, r, "/import/"+draftID, http.StatusSeeOther)
}
func (h *UploadHandler) ImportGet(w http.ResponseWriter, r *http.Request) {
draftID := chi.URLParam(r, "id")
userID := auth.UserFromCtx(r.Context()).ID
draft, err := h.repo.GetDraftForUser(draftID, userID)
if err != nil {
HTTPError(w, http.StatusNotFound)
return
}
data := BaseData(h.auth, r)
data["Draft"] = draft
h.render.Render(w, http.StatusOK, "import", data)
}
func (h *UploadHandler) ImportPost(w http.ResponseWriter, r *http.Request) {
draftID := chi.URLParam(r, "id")
userID := auth.UserFromCtx(r.Context()).ID
if !h.auth.CheckCSRF(r) {
HTTPError(w, http.StatusForbidden)
return
}
draft, err := h.repo.GetDraftForUser(draftID, userID)
if err != nil {
HTTPError(w, http.StatusNotFound)
return
}
source := strings.TrimSpace(r.FormValue("source"))
var imported, skipped int
for i, dq := range draft.Questions {
if r.FormValue(fmt.Sprintf("delete_%d", i)) == "on" {
skipped++
continue
}
text := strings.TrimSpace(r.FormValue(fmt.Sprintf("q_text_%d", i)))
if text == "" {
skipped++
continue
}
correctIdx := r.FormValue(fmt.Sprintf("correct_%d", i))
var answers []*models.Answer
for j := range dq.Answers {
aText := strings.TrimSpace(r.FormValue(fmt.Sprintf("a_text_%d_%d", i, j)))
if aText == "" {
continue
}
answers = append(answers, &models.Answer{
Text: aText,
IsCorrect: fmt.Sprintf("%d", j) == correctIdx,
})
}
var nCorrect int
for _, a := range answers {
if a.IsCorrect {
nCorrect++
}
}
if len(answers) < 2 || nCorrect != 1 {
skipped++
continue
}
q := &models.Question{Text: text, Source: source}
if err := h.repo.InsertQuestion(q, answers); err != nil {
slog.Error("insert question", "err", err)
skipped++
continue
}
imported++
}
if err := h.repo.DeleteDraft(draftID); err != nil {
slog.Error("delete draft", "id", draftID, "err", err)
}
h.auth.SetFlash(r, fmt.Sprintf("Imported %d question(s), %d skipped.", imported, skipped))
http.Redirect(w, r, "/", http.StatusSeeOther)
}
+19
View File
@@ -44,6 +44,25 @@ type TestAnswer struct {
AnsweredAt sql.NullTime
}
// Draft holds LLM-extracted questions pending user review before import.
type Draft struct {
ID string
UserID int64
Source string
Questions []DraftQuestion
CreatedAt time.Time
}
type DraftQuestion struct {
Text string `json:"text"`
Answers []DraftAnswer `json:"answers"`
}
type DraftAnswer struct {
Text string `json:"text"`
IsCorrect bool `json:"is_correct"`
}
type UserQuestionStat struct {
UserID int64
QuestionID string
+81
View File
@@ -0,0 +1,81 @@
{{define "content"}}
<div class="mb-6 flex items-start justify-between gap-4">
<div>
<h1 class="text-2xl font-bold text-gray-800">Review Import</h1>
<p class="text-sm text-gray-500 mt-1">
{{len .Draft.Questions}} question(s) found.
Edit any text, choose the correct answer, then confirm.
Check "Delete" to skip a question.
</p>
</div>
<a href="/upload" class="text-sm text-gray-500 hover:underline whitespace-nowrap mt-1">← Upload another</a>
</div>
{{if eq (len .Draft.Questions) 0}}
<div class="text-center py-16 text-gray-400">
<p class="text-lg">No questions were found in this document.</p>
<p class="text-sm mt-2">The file may not contain multiple-choice questions, or the LLM was unable to extract them.</p>
<a href="/upload" class="mt-6 inline-block text-blue-600 hover:underline text-sm">Try another file</a>
</div>
{{else}}
<form method="post" action="/import/{{.Draft.ID}}" class="space-y-6">
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
<div class="flex items-center gap-3">
<label class="text-sm font-medium text-gray-700 whitespace-nowrap">Source</label>
<input type="text" name="source" value="{{.Draft.Source}}"
class="flex-1 border border-gray-300 rounded-md px-3 py-1.5 text-sm
focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500">
</div>
{{range $i, $q := .Draft.Questions}}
<div class="border border-gray-200 rounded-lg overflow-hidden shadow-sm">
<div class="bg-gray-50 border-b border-gray-200 px-4 py-2 flex items-center justify-between">
<span class="text-xs font-semibold text-gray-500 uppercase tracking-wide">
Question {{inc $i}}
</span>
<label class="flex items-center gap-2 text-sm text-red-600 cursor-pointer select-none">
<input type="checkbox" name="delete_{{$i}}"
class="rounded border-gray-300 text-red-500 focus:ring-red-400">
Delete
</label>
</div>
<div class="p-4 space-y-3">
<textarea name="q_text_{{$i}}" rows="3" required
class="w-full border border-gray-300 rounded-md px-3 py-2 text-sm shadow-sm
focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500
resize-y">{{$q.Text}}</textarea>
<div class="space-y-2">
{{range $j, $a := $q.Answers}}
<div class="flex items-center gap-3">
<input type="radio" name="correct_{{$i}}" value="{{$j}}"
{{if $a.IsCorrect}}checked{{end}}
class="text-blue-600 focus:ring-blue-500 flex-shrink-0">
<input type="text" name="a_text_{{$i}}_{{$j}}" value="{{$a.Text}}" required
class="flex-1 border border-gray-300 rounded-md px-3 py-1.5 text-sm shadow-sm
focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500">
</div>
{{end}}
</div>
<p class="text-xs text-gray-400">Select the radio button next to the correct answer.</p>
</div>
</div>
{{end}}
<div class="flex gap-3 pt-2 pb-8">
<button type="submit"
class="flex-1 bg-blue-600 hover:bg-blue-700 text-white py-2.5 px-4 rounded-md
text-sm font-semibold shadow-sm">
Confirm import
</button>
<a href="/upload"
class="px-4 py-2.5 border border-gray-300 rounded-md text-sm font-medium text-gray-700
hover:bg-gray-50 text-center">
Cancel
</a>
</div>
</form>
{{end}}
{{end}}
+42
View File
@@ -0,0 +1,42 @@
{{define "content"}}
<div class="max-w-lg">
<h1 class="text-2xl font-bold text-gray-800 mb-1">Upload Document</h1>
<p class="text-sm text-gray-500 mb-6">Accepts PDF or DOCX. Max 20 MB.</p>
{{if .Error}}
<div class="mb-5 text-sm text-red-700 bg-red-50 border border-red-200 px-4 py-3 rounded-md">
{{.Error}}
</div>
{{end}}
<form method="post" action="/upload" enctype="multipart/form-data" class="space-y-5">
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
<div>
<label class="block text-sm font-medium text-gray-700 mb-1">File</label>
<input type="file" name="file" accept=".pdf,.docx" required
class="block w-full text-sm text-gray-700
file:mr-3 file:py-2 file:px-4
file:rounded-md file:border-0
file:text-sm file:font-medium
file:bg-blue-50 file:text-blue-700
hover:file:bg-blue-100">
</div>
<div>
<label class="block text-sm font-medium text-gray-700 mb-1">
Source name <span class="text-gray-400 font-normal">(optional — defaults to filename)</span>
</label>
<input type="text" name="source" placeholder="e.g. Chapter 3"
class="w-full border border-gray-300 rounded-md px-3 py-2 text-sm shadow-sm
focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500">
</div>
<button type="submit"
class="w-full bg-blue-600 hover:bg-blue-700 text-white py-2.5 px-4 rounded-md
text-sm font-semibold shadow-sm">
Extract questions
</button>
</form>
</div>
{{end}}