aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore1
-rw-r--r--COPYING13
-rw-r--r--Makefile13
-rw-r--r--README.md59
-rw-r--r--autograde-Makefile10
-rw-r--r--skel/.gitignore2
-rw-r--r--skel/COPYING11
-rw-r--r--skel/Makefile5
-rw-r--r--skel/checks.sig23
-rw-r--r--skel/checks.sml36
-rw-r--r--skel/helper.sig61
-rw-r--r--skel/helper.sml173
-rw-r--r--skel/main.sml25
-rwxr-xr-xskel/run.sh26
-rw-r--r--skel/sources.cm9
-rw-r--r--skel/support/README1
-rw-r--r--skel/support/sources.cm2
-rwxr-xr-xskel/test_checks.sh16
-rw-r--r--skel/test_handins/README5
19 files changed, 491 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ca6d64e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+autograde.tar
diff --git a/COPYING b/COPYING
new file mode 100644
index 0000000..f976e83
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,13 @@
+Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu>
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..f73845b
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,13 @@
+#!/usr/bin/make
+# Targets:
+# make DIRNAME-autograde
+# enter DIRNAME and call the autograde target to generate
+# ./autograde.tar
+
+.PHONY: clean
+
+%-autograde:
+ $(MAKE) -C $* autograde
+
+clean:
+ rm -f *.tar
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..015a19f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,59 @@
+This is a skeleton for a checkscript for autolab. In order to make
+this happen there are a small battery of SML scripts.
+
+New in f17 for 15-317: we introduce a directory per assignment
+containing the autolab checks, rather than using a single "src"
+directory that gets modified every time we have a new assignment.
+
+The central one is `checks.sml`. In it, you can describe a sequence of
+checks to perform (not necessarily related to any given problem). You
+can also describe a series of Autolab problems to grade. These checks
+and problems belong in the variable `checks`. See the file for
+examples and further documentation. When writing your checks, you
+should assume that a given student's files are in the subdirectory
+`./handin` (relative to the file `main.sml`) during check execution.
+
+This file is loaded by the file `main.sml`. This file runs the tests
+described in the the `checks` variable from `checks.sml` and outputs a
+correctly-formatted [Autolab score string]. You should never need to
+edit main.sml.
+
+The file `helper.sml` provides a variety of helper functions for
+grading, described
+
+In order to create an autolab test for homework NN, we recommend you
+do the following:
+
+ - Copy `./skel/*` to `./NN/`
+ - If you want to check your students submitted the correct files, use
+ the checkFilesExist check described in the skeleton `checks.sml`.
+ It will check that each file in the list exists under the directory
+ `./handin`, and will abort the grading if it is missing.
+ - To keep things organised, put various utilities, etc., you need to
+ grade assignments under `./NN/support/`.
+ - If you're doing anything really really funky, update the
+ `autograde` target in `./NN/Makefile` . It **must** generate the
+ file `./autograde.tar` containing all files you require for your
+ checks. The skeleton Makefile at the time of this writing simply
+ creates a tarball with the contents of `./NN/*` in the top level,
+ and dereferences any symlinks.
+ - Test your scripts using `./NN/test_checks.sh` (see below).
+ - Run `make NN-autograde` to generate `autograder.tar`.
+ - Upload `./autograder.tar` and `autograde-Makefile` to Autolab.
+
+And you're all set.
+
+If you want to automatically test your `./NN/checks.sml` against some
+submissions to make sure the output scores are reasonable, the script
+`./NN/test_checks.sh` will look in the directory `./NN/test_handins/`
+for test submissions, and will run your check scripts against them.
+See `./NN/test_handins/README` for more details.
+
+I know of no *clean* way to abort the autograder without updating any
+scores. The semantically cleanest way is to return an empty scores
+dictionary in the autoscore. However, at the time of this writing,
+Autolab complains with an error when we do so. This is, in my opinion,
+a bug, and should be fixed by this [pull request].
+
+[Autolab score string]: https://autolab.github.io/docs/lab/
+[pull request]: https://github.com/autolab/Autolab/pull/895
diff --git a/autograde-Makefile b/autograde-Makefile
new file mode 100644
index 0000000..e05a823
--- /dev/null
+++ b/autograde-Makefile
@@ -0,0 +1,10 @@
+# This file is called in a VM by the autograder
+all:
+ tar -mxf autograde.tar
+ # We are requied by AutoLab to always terminate
+ # by printing a valid score string. If ./run.sh
+ # exists with an error, this will not happen.
+ ./run.sh || \
+ ( echo "run.sh exited with non-zero exit code.";\
+ echo "Contact course staff."; \
+ echo "{\"score\": {}}" )
diff --git a/skel/.gitignore b/skel/.gitignore
new file mode 100644
index 0000000..8747365
--- /dev/null
+++ b/skel/.gitignore
@@ -0,0 +1,2 @@
+grader.*
+.cm/*
diff --git a/skel/COPYING b/skel/COPYING
new file mode 100644
index 0000000..c8ca2c9
--- /dev/null
+++ b/skel/COPYING
@@ -0,0 +1,11 @@
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
diff --git a/skel/Makefile b/skel/Makefile
new file mode 100644
index 0000000..c6a5adb
--- /dev/null
+++ b/skel/Makefile
@@ -0,0 +1,5 @@
+autograde:
+ # h: follow links, i.e., so that we include the contents of the tutch symlink
+ tar chf ../autograde.tar --exclude=./Makefile --exclude=test_handins .
+
+.PHONY: autograde
diff --git a/skel/checks.sig b/skel/checks.sig
new file mode 100644
index 0000000..6b0453c
--- /dev/null
+++ b/skel/checks.sig
@@ -0,0 +1,23 @@
+(* Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu> *)
+(* Distributed under the ISC license, see COPYING for details. *)
+
+signature CHECKS =
+sig
+ (* checks are either: *)
+ (* Check (name, f): *)
+ (* f : checks some property, and can catastrophically abort *)
+ (* or raise some exception if something is not satisfied. *)
+ (* Useful for sanity checks like: do all desired files *)
+ (* exist? *)
+ (* Problem (name, f): *)
+ (* name : must be the same name as the autolab problem *)
+ (* that is being graded. *)
+ (* f : returns an integer, which is the score awarded for *)
+ (* problem "name". *)
+ datatype checks = Check of string * (unit -> unit)
+ | Problem of string * (unit -> real)
+
+ val checks : checks list
+
+ val scoreboard : (string * real) list -> int list option
+end
diff --git a/skel/checks.sml b/skel/checks.sml
new file mode 100644
index 0000000..a62efe8
--- /dev/null
+++ b/skel/checks.sml
@@ -0,0 +1,36 @@
+(* Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu> *)
+(* Distributed under the ISC license, see COPYING for details. *)
+
+functor ChecksHelper (structure H : HELPER) : CHECKS where type checks = H.checks =
+struct
+
+datatype checks = datatype H.checks
+
+(*****************************************************)
+(********** CONFIGURE ME HERE *******)
+(*****************************************************)
+
+val requiredFiles = ["hw0.pdf", "ex1.tut", "ex2.tut"]
+
+(* Returns the score l + h *)
+fun check1 l h : real = l + h
+
+(* Is evil and always gives the student 0.0 *)
+fun check2 () = 0.0
+
+(* We first make sure all of the required files exist. *)
+(* We then grade AutoLab problem "ex1" using the test1 function. *)
+(* Finally, we grade AutoLab problem "ex2". *)
+val checks = [ H.Check ("all files present", fn _ => H.checkFilesExist requiredFiles)
+ , H.Check ("hw0.pdf", fn _ => H.checkPDF "hw0.pdf")
+ , H.Problem ("ex1", fn _ => check1 3.0 4.0)
+ , H.Problem ("ex2", fn _ => check2 ()) ]
+
+(* Empty scoreboard *)
+fun scoreboard _ = NONE
+
+(*****************************************************)
+(********** END CONFIGURATION *******)
+(*****************************************************)
+
+end
diff --git a/skel/helper.sig b/skel/helper.sig
new file mode 100644
index 0000000..82467d6
--- /dev/null
+++ b/skel/helper.sig
@@ -0,0 +1,61 @@
+(* Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu> *)
+(* Distributed under the ISC license, see COPYING for details. *)
+
+signature HELPER =
+sig
+ datatype checks = Check of string * (unit -> unit)
+ | Problem of string * (unit -> real)
+
+ (* Path to the directory under which we can find a student's *)
+ (* submitted files. *)
+ val handinPath : string
+
+ (* Takes in the name of a file submitted by a student *)
+ (* and joins it with handinPath. *)
+ val joinHandinPath : string -> string
+
+ (* Removes handinPath from a path if it prefixes it. *)
+ val stripHandinPath : string -> string
+
+ (* Takes in a list of filenames, and checks if those files *)
+ (* exist in the handinPath directory. *)
+ (* Aborts catastrophically if a file is missing. *)
+ val checkFilesExist : string list -> unit
+
+ (* Uses ghostscript to check if the student submitted a valid *)
+ (* PDF (or an empty file in its place). Aborts catastrophically *)
+ (* if missing. *)
+ val checkPDF : string -> unit
+
+ (* Runs a series of checks and then outputs a list of *)
+ (* (problem name, score) tuples *)
+ val runChecks : checks list -> (string * real) list
+
+ (* Produces an AutoLab JSON score string from a scores list *)
+ (* and optional scoreboard *)
+ val scoresToString : (string * real) list * int list option -> string
+
+ (* Puts a list of strings in a box for printing. *)
+ val stringsInBox : string list -> string list
+
+ (* Prints a list of strings in a box. *)
+ val printInBox : string list -> unit
+
+ (* Prints a string followed by newline. *)
+ val printLn : string -> unit
+
+ (* printLns a list of strings, then the empty score, *)
+ (* then exits. Useful when you need to abort early. *)
+ val abortWithMessage : string list -> 'a
+
+ (* Prints a command and then runs it, returning the exit status *)
+ val runCmd : string -> OS.Process.status
+
+ (* Runs a command (with argument list) using Posix.Process.execp. *)
+ (* Return the program's output as a string, along with its exit status. *)
+ val execpOutput : string * string list -> string * Posix.Process.exit_status
+
+ (* Takes a regex in awk format, and a string, and checks if *)
+ (* the regex matches the string *)
+ val matchesAwkRegex : string * string -> bool
+end
diff --git a/skel/helper.sml b/skel/helper.sml
new file mode 100644
index 0000000..d20e96f
--- /dev/null
+++ b/skel/helper.sml
@@ -0,0 +1,173 @@
+(* Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu> *)
+(* Distributed under the ISC license, see COPYING for details. *)
+
+functor Helper (structure C : sig val handinPath : string end)
+ :> HELPER =
+struct
+
+exception MissingFile of string
+
+datatype checks = Check of string * (unit -> unit)
+ | Problem of string * (unit -> real)
+
+val handinPath = C.handinPath
+
+fun printLn s = print (s ^ "\n")
+
+(* Puts strings in boxes... *)
+fun stringsInBox strs =
+ let fun repeatChar c n = String.implode (List.tabulate (n, fn _ => c))
+ val maxLength = List.foldl (fn (s,themax) => Int.max (String.size s, themax)) 0 strs
+ val edge = repeatChar #"#" (maxLength + 4)
+ in
+ edge :: (List.foldr (fn (s,thelist) => ("# "
+ ^ s
+ ^ (repeatChar #" " (maxLength - (String.size s)))
+ ^ " #") :: thelist)
+ [edge]
+ strs)
+ end
+
+(* Prints strings in boxes... *)
+fun printInBox strs = List.app printLn (stringsInBox strs)
+
+(* Generates an AutoLab json score string *)
+fun scoresToString (scores, scoreboard) =
+ let val scoreStrings = map (fn (problem, score) => "\"" ^ problem ^ "\": " ^ (Real.toString score)) scores
+ val scores = String.concatWith ", " scoreStrings
+ val scoreboard = case scoreboard
+ of SOME l => ", \"scoreboard\": [" ^ (String.concatWith ", " (map Int.toString l)) ^ "]"
+ | NONE => ""
+ in
+ "{\"scores\": {" ^ scores ^ "}" ^ scoreboard ^ "}"
+ end
+
+(* Aborts the program by printing strs, and gives an empty score. *)
+fun abortWithMessage strs =
+ let val _ = List.app printLn strs
+ val _ = printLn (scoresToString ([],NONE))
+ in
+ OS.Process.exit OS.Process.success
+ end
+
+
+(* Reads the lines of a file into a list *)
+(* Each string in the file will always be contain a newline (#"\n") at the end. *)
+fun readLines filename =
+ let val inFile = TextIO.openIn filename
+ fun readlines ins =
+ case TextIO.inputLine ins
+ of SOME ln => ln :: readlines ins
+ | NONE => []
+ val lines = readlines inFile
+ val _ = TextIO.closeIn inFile
+ in
+ lines
+ end
+
+(* Check if the file exists *)
+fun checkFileExists (name : string) : unit =
+ if OS.FileSys.access (name, [OS.FileSys.A_READ])
+ then ()
+ else raise MissingFile name
+
+fun joinHandinPath file =
+ OS.Path.concat (handinPath, file)
+
+fun stripHandinPath path =
+ if String.isPrefix handinPath path then
+ String.extract (path, String.size handinPath + 1, NONE)
+ else
+ path
+
+
+(* Takes in a list of filenames, and checks if those files *)
+(* exist in the handinPath directory. *)
+(* Exits catastrophically if a file is missing. *)
+fun checkFilesExist filenames =
+ List.app (checkFileExists o joinHandinPath) filenames
+ handle MissingFile name => (abortWithMessage o stringsInBox)
+ [ "File " ^ (stripHandinPath name) ^ " missing."
+ , "Please make sure you included all required files and resubmit."]
+
+fun runCmd cmd = (printLn cmd; OS.Process.system cmd)
+
+(* Reads from fd in n byte chunks and treats it all as strings. *)
+fun readAllFDAsString (fd, n) =
+ let val v = Posix.IO.readVec (fd, n)
+ in if Word8Vector.length v = 0 then
+ ""
+ else
+ (Byte.bytesToString v) ^ (readAllFDAsString (fd, n))
+ end
+
+(* Runs a command c (command and argument list) using Posix.Process.execp. *)
+(* Return the program's output as a string, along with its exit status. *)
+fun execpOutput (c : string * string list) : string * Posix.Process.exit_status =
+ let val { infd = infd, outfd = outfd } = Posix.IO.pipe ()
+ in case Posix.Process.fork ()
+ of NONE => (* Child *)
+ (( Posix.IO.close infd
+ ; Posix.IO.dup2 { old = outfd, new = Posix.FileSys.stdout }
+ ; Posix.IO.dup2 { old = outfd, new = Posix.FileSys.stderr }
+ ; Posix.Process.execp c)
+ handle OS.SysErr (err, _) =>
+ ( print ("Fatal error in child: " ^ err ^ "\n")
+ ; OS.Process.exit OS.Process.failure ))
+ | SOME pid => (* Parent *)
+ let val _ = Posix.IO.close outfd
+ val (_, status) = Posix.Process.waitpid (Posix.Process.W_CHILD pid, [])
+ val output = readAllFDAsString (infd, 100)
+ val _ = Posix.IO.close infd
+ in (output, status) end
+ end
+
+
+(* Check if a submitted file is a valid PDF using ghostscript. *)
+fun checkPDF pdf =
+ let val spdf = joinHandinPath pdf in
+ case ( runCmd ("gs -o/dev/null -sDEVICE=nullpage " ^ spdf)
+ , Posix.FileSys.ST.size (Posix.FileSys.stat spdf) )
+ of (_,0) => let val _ = printInBox [ "Warning: The empty file " ^ pdf ^ " is not a valid PDF document."
+ , "Please make sure to resubmit with a valid PDF document in its place." ]
+ in () end
+ | (0,_) => ()
+ | _ => (abortWithMessage o stringsInBox)
+ [ "The file " ^ pdf ^ " is not a valid PDF document."
+ , "Please resubmit with a valid PDF document (or an empty file) in its place."
+ , "If you are convinced you submitted a valid PDF, please contact the course staff." ]
+ end
+
+(* Runs all of the checks and grades all of the problems in "checks". *)
+fun runChecks (checks : checks list) =
+ List.foldl (fn (cs,results) =>
+ case cs
+ of (Check (n, c)) => let val _ = printLn ("\n\nRunning check " ^ n ^ "...")
+ val _ = c ()
+ val _ = printLn " Success.\n"
+ in results end
+ | (Problem (n, c)) => let val _ = printLn ("\n\nChecking problem " ^ n ^ "...")
+ val res = c ()
+ val _ = printLn (" Score: " ^ (Real.toString res) ^ ".\n")
+ in (n, res) :: results end)
+ []
+ checks
+
+
+(* Returns a score of zero for all problems. *)
+(* Useful when you need to abort but still provide a score. *)
+fun failAll (checks : checks list) =
+ List.foldr (fn (cs,results) =>
+ case cs
+ of Problem (n, c) => (n, 0) :: results
+ | _ => results)
+ []
+ checks
+
+structure RE = RegExpFn(structure P = AwkSyntax
+ structure E = ThompsonEngine)
+
+fun matchesAwkRegex (r, s) =
+ let val r = RE.find (RE.compileString r)
+ in Option.isSome (StringCvt.scanString r s) end
+end
diff --git a/skel/main.sml b/skel/main.sml
new file mode 100644
index 0000000..95e5bb5
--- /dev/null
+++ b/skel/main.sml
@@ -0,0 +1,25 @@
+(* Copyright (C) 2017 Ryan Kavanagh <rkavanagh@cs.cmu.edu> *)
+(* Distributed under the ISC license, see COPYING for details. *)
+
+structure Main :>
+ sig
+ val main : (string * string list) -> OS.Process.status
+ end
+=
+struct
+structure H = Helper (structure C = struct val handinPath = "handin" end)
+structure C = ChecksHelper(structure H = H)
+
+fun main _ =
+ let val scores = H.runChecks C.checks
+ val scoreboard = C.scoreboard scores
+ (* Make sure the scores are on the last line. *)
+ val _ = print "\n\n\n"
+ (* This below *must must must* be the last thing printed.. *)
+ val _ = H.printLn (H.scoresToString (scores, scoreboard))
+ in OS.Process.success end
+ handle _ => (H.abortWithMessage o H.stringsInBox)
+ [ "Experienced uncaught exception!"
+ , "If you believe this to be in error, please contact your"
+ ^ " course staff." ]
+end
diff --git a/skel/run.sh b/skel/run.sh
new file mode 100755
index 0000000..ff313b2
--- /dev/null
+++ b/skel/run.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+abort () {
+ if [ "x$1" != "x" ]; then
+ echo "$1"
+ fi
+ printf "\n\n{\"scores\": {}}"
+ exit 0
+}
+
+tar -mxf autograde.tar || \
+ abort "Failed to extract autograder."
+
+[ -d handin ] || mkdir handin
+
+[ -f handin.tar ] || \
+ abort "Submission is expected to be at handin.tar. This file does not exist!"
+
+tar -xf handin.tar -C handin || \
+ abort "Failed to extract submission."
+
+ml-build sources.cm Main.main grader || \
+ abort "Failed to compile autograder. Please contact course staff."
+
+sml @SMLload grader.* || \
+ abort "SML autograder exited with error. Please contact course staff."
diff --git a/skel/sources.cm b/skel/sources.cm
new file mode 100644
index 0000000..7359047
--- /dev/null
+++ b/skel/sources.cm
@@ -0,0 +1,9 @@
+Group is
+ $/basis.cm
+ $/regexp-lib.cm
+ support/sources.cm
+ helper.sig (* the HELPER signature *)
+ helper.sml (* the Helper module *)
+ checks.sig (* the CHECKS signature *)
+ checks.sml (* the ChecksHelper functor *)
+ main.sml (* the big cheese *)
diff --git a/skel/support/README b/skel/support/README
new file mode 100644
index 0000000..8c31e1a
--- /dev/null
+++ b/skel/support/README
@@ -0,0 +1 @@
+Put any support files here.
diff --git a/skel/support/sources.cm b/skel/support/sources.cm
new file mode 100644
index 0000000..fad8872
--- /dev/null
+++ b/skel/support/sources.cm
@@ -0,0 +1,2 @@
+(* Add any extra support SML files here *)
+Group is
diff --git a/skel/test_checks.sh b/skel/test_checks.sh
new file mode 100755
index 0000000..975ea4c
--- /dev/null
+++ b/skel/test_checks.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+make -s autograde
+
+for f in test_handins/*.tar; do
+ base=`basename -s .tar ${f}`
+ tmp=`mktemp -d`
+ cp ../autograde.tar ${tmp}
+ cp test_handins/${base}.tar ${tmp}/handin.tar
+ cp ../autograde-Makefile ${tmp}
+ echo "\e[41m!!!!!!! Last 10 output lines for test ${base}:\e[0m"
+ (cd ${tmp}; make -s -f autograde-Makefile | tail -n 10)
+ echo "\e[42m!!!!!!! Expected result:\e[0m"
+ cat test_handins/${base}.exp
+ rm -fr ${tmp}
+done
diff --git a/skel/test_handins/README b/skel/test_handins/README
new file mode 100644
index 0000000..b43561b
--- /dev/null
+++ b/skel/test_handins/README
@@ -0,0 +1,5 @@
+To test your checks.sml, place sample submissions in this directory
+according to the following format:
+
+ somename.tar : the submission you want to test
+ somename.exp : the expected json score
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 23:19:07 +0000