From 5e78c729407999bd26af4d446edf0edf7d0af94e Mon Sep 17 00:00:00 2001 From: Ryan Kavanagh Date: Thu, 13 Sep 2018 13:09:35 -0400 Subject: Import autograder from 15-317 f17 --- skel/.gitignore | 2 + skel/COPYING | 11 +++ skel/Makefile | 5 ++ skel/checks.sig | 23 +++++++ skel/checks.sml | 36 ++++++++++ skel/helper.sig | 61 +++++++++++++++++ skel/helper.sml | 173 +++++++++++++++++++++++++++++++++++++++++++++++ skel/main.sml | 25 +++++++ skel/run.sh | 26 +++++++ skel/sources.cm | 9 +++ skel/support/README | 1 + skel/support/sources.cm | 2 + skel/test_checks.sh | 16 +++++ skel/test_handins/README | 5 ++ 14 files changed, 395 insertions(+) create mode 100644 skel/.gitignore create mode 100644 skel/COPYING create mode 100644 skel/Makefile create mode 100644 skel/checks.sig create mode 100644 skel/checks.sml create mode 100644 skel/helper.sig create mode 100644 skel/helper.sml create mode 100644 skel/main.sml create mode 100755 skel/run.sh create mode 100644 skel/sources.cm create mode 100644 skel/support/README create mode 100644 skel/support/sources.cm create mode 100755 skel/test_checks.sh create mode 100644 skel/test_handins/README (limited to 'skel') diff --git a/skel/.gitignore b/skel/.gitignore new file mode 100644 index 0000000..8747365 --- /dev/null +++ b/skel/.gitignore @@ -0,0 +1,2 @@ +grader.* +.cm/* diff --git a/skel/COPYING b/skel/COPYING new file mode 100644 index 0000000..c8ca2c9 --- /dev/null +++ b/skel/COPYING @@ -0,0 +1,11 @@ +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/skel/Makefile b/skel/Makefile new file mode 100644 index 0000000..c6a5adb --- /dev/null +++ b/skel/Makefile @@ -0,0 +1,5 @@ +autograde: + # h: follow links, i.e., so that we include the contents of the tutch symlink + tar chf ../autograde.tar --exclude=./Makefile --exclude=test_handins . + +.PHONY: autograde diff --git a/skel/checks.sig b/skel/checks.sig new file mode 100644 index 0000000..6b0453c --- /dev/null +++ b/skel/checks.sig @@ -0,0 +1,23 @@ +(* Copyright (C) 2017 Ryan Kavanagh *) +(* Distributed under the ISC license, see COPYING for details. *) + +signature CHECKS = +sig + (* checks are either: *) + (* Check (name, f): *) + (* f : checks some property, and can catastrophically abort *) + (* or raise some exception if something is not satisfied. *) + (* Useful for sanity checks like: do all desired files *) + (* exist? *) + (* Problem (name, f): *) + (* name : must be the same name as the autolab problem *) + (* that is being graded. *) + (* f : returns an integer, which is the score awarded for *) + (* problem "name". *) + datatype checks = Check of string * (unit -> unit) + | Problem of string * (unit -> real) + + val checks : checks list + + val scoreboard : (string * real) list -> int list option +end diff --git a/skel/checks.sml b/skel/checks.sml new file mode 100644 index 0000000..a62efe8 --- /dev/null +++ b/skel/checks.sml @@ -0,0 +1,36 @@ +(* Copyright (C) 2017 Ryan Kavanagh *) +(* Distributed under the ISC license, see COPYING for details. *) + +functor ChecksHelper (structure H : HELPER) : CHECKS where type checks = H.checks = +struct + +datatype checks = datatype H.checks + +(*****************************************************) +(********** CONFIGURE ME HERE *******) +(*****************************************************) + +val requiredFiles = ["hw0.pdf", "ex1.tut", "ex2.tut"] + +(* Returns the score l + h *) +fun check1 l h : real = l + h + +(* Is evil and always gives the student 0.0 *) +fun check2 () = 0.0 + +(* We first make sure all of the required files exist. *) +(* We then grade AutoLab problem "ex1" using the test1 function. *) +(* Finally, we grade AutoLab problem "ex2". *) +val checks = [ H.Check ("all files present", fn _ => H.checkFilesExist requiredFiles) + , H.Check ("hw0.pdf", fn _ => H.checkPDF "hw0.pdf") + , H.Problem ("ex1", fn _ => check1 3.0 4.0) + , H.Problem ("ex2", fn _ => check2 ()) ] + +(* Empty scoreboard *) +fun scoreboard _ = NONE + +(*****************************************************) +(********** END CONFIGURATION *******) +(*****************************************************) + +end diff --git a/skel/helper.sig b/skel/helper.sig new file mode 100644 index 0000000..82467d6 --- /dev/null +++ b/skel/helper.sig @@ -0,0 +1,61 @@ +(* Copyright (C) 2017 Ryan Kavanagh *) +(* Distributed under the ISC license, see COPYING for details. *) + +signature HELPER = +sig + datatype checks = Check of string * (unit -> unit) + | Problem of string * (unit -> real) + + (* Path to the directory under which we can find a student's *) + (* submitted files. *) + val handinPath : string + + (* Takes in the name of a file submitted by a student *) + (* and joins it with handinPath. *) + val joinHandinPath : string -> string + + (* Removes handinPath from a path if it prefixes it. *) + val stripHandinPath : string -> string + + (* Takes in a list of filenames, and checks if those files *) + (* exist in the handinPath directory. *) + (* Aborts catastrophically if a file is missing. *) + val checkFilesExist : string list -> unit + + (* Uses ghostscript to check if the student submitted a valid *) + (* PDF (or an empty file in its place). Aborts catastrophically *) + (* if missing. *) + val checkPDF : string -> unit + + (* Runs a series of checks and then outputs a list of *) + (* (problem name, score) tuples *) + val runChecks : checks list -> (string * real) list + + (* Produces an AutoLab JSON score string from a scores list *) + (* and optional scoreboard *) + val scoresToString : (string * real) list * int list option -> string + + (* Puts a list of strings in a box for printing. *) + val stringsInBox : string list -> string list + + (* Prints a list of strings in a box. *) + val printInBox : string list -> unit + + (* Prints a string followed by newline. *) + val printLn : string -> unit + + (* printLns a list of strings, then the empty score, *) + (* then exits. Useful when you need to abort early. *) + val abortWithMessage : string list -> 'a + + (* Prints a command and then runs it, returning the exit status *) + val runCmd : string -> OS.Process.status + + (* Runs a command (with argument list) using Posix.Process.execp. *) + (* Return the program's output as a string, along with its exit status. *) + val execpOutput : string * string list -> string * Posix.Process.exit_status + + (* Takes a regex in awk format, and a string, and checks if *) + (* the regex matches the string *) + val matchesAwkRegex : string * string -> bool +end diff --git a/skel/helper.sml b/skel/helper.sml new file mode 100644 index 0000000..d20e96f --- /dev/null +++ b/skel/helper.sml @@ -0,0 +1,173 @@ +(* Copyright (C) 2017 Ryan Kavanagh *) +(* Distributed under the ISC license, see COPYING for details. *) + +functor Helper (structure C : sig val handinPath : string end) + :> HELPER = +struct + +exception MissingFile of string + +datatype checks = Check of string * (unit -> unit) + | Problem of string * (unit -> real) + +val handinPath = C.handinPath + +fun printLn s = print (s ^ "\n") + +(* Puts strings in boxes... *) +fun stringsInBox strs = + let fun repeatChar c n = String.implode (List.tabulate (n, fn _ => c)) + val maxLength = List.foldl (fn (s,themax) => Int.max (String.size s, themax)) 0 strs + val edge = repeatChar #"#" (maxLength + 4) + in + edge :: (List.foldr (fn (s,thelist) => ("# " + ^ s + ^ (repeatChar #" " (maxLength - (String.size s))) + ^ " #") :: thelist) + [edge] + strs) + end + +(* Prints strings in boxes... *) +fun printInBox strs = List.app printLn (stringsInBox strs) + +(* Generates an AutoLab json score string *) +fun scoresToString (scores, scoreboard) = + let val scoreStrings = map (fn (problem, score) => "\"" ^ problem ^ "\": " ^ (Real.toString score)) scores + val scores = String.concatWith ", " scoreStrings + val scoreboard = case scoreboard + of SOME l => ", \"scoreboard\": [" ^ (String.concatWith ", " (map Int.toString l)) ^ "]" + | NONE => "" + in + "{\"scores\": {" ^ scores ^ "}" ^ scoreboard ^ "}" + end + +(* Aborts the program by printing strs, and gives an empty score. *) +fun abortWithMessage strs = + let val _ = List.app printLn strs + val _ = printLn (scoresToString ([],NONE)) + in + OS.Process.exit OS.Process.success + end + + +(* Reads the lines of a file into a list *) +(* Each string in the file will always be contain a newline (#"\n") at the end. *) +fun readLines filename = + let val inFile = TextIO.openIn filename + fun readlines ins = + case TextIO.inputLine ins + of SOME ln => ln :: readlines ins + | NONE => [] + val lines = readlines inFile + val _ = TextIO.closeIn inFile + in + lines + end + +(* Check if the file exists *) +fun checkFileExists (name : string) : unit = + if OS.FileSys.access (name, [OS.FileSys.A_READ]) + then () + else raise MissingFile name + +fun joinHandinPath file = + OS.Path.concat (handinPath, file) + +fun stripHandinPath path = + if String.isPrefix handinPath path then + String.extract (path, String.size handinPath + 1, NONE) + else + path + + +(* Takes in a list of filenames, and checks if those files *) +(* exist in the handinPath directory. *) +(* Exits catastrophically if a file is missing. *) +fun checkFilesExist filenames = + List.app (checkFileExists o joinHandinPath) filenames + handle MissingFile name => (abortWithMessage o stringsInBox) + [ "File " ^ (stripHandinPath name) ^ " missing." + , "Please make sure you included all required files and resubmit."] + +fun runCmd cmd = (printLn cmd; OS.Process.system cmd) + +(* Reads from fd in n byte chunks and treats it all as strings. *) +fun readAllFDAsString (fd, n) = + let val v = Posix.IO.readVec (fd, n) + in if Word8Vector.length v = 0 then + "" + else + (Byte.bytesToString v) ^ (readAllFDAsString (fd, n)) + end + +(* Runs a command c (command and argument list) using Posix.Process.execp. *) +(* Return the program's output as a string, along with its exit status. *) +fun execpOutput (c : string * string list) : string * Posix.Process.exit_status = + let val { infd = infd, outfd = outfd } = Posix.IO.pipe () + in case Posix.Process.fork () + of NONE => (* Child *) + (( Posix.IO.close infd + ; Posix.IO.dup2 { old = outfd, new = Posix.FileSys.stdout } + ; Posix.IO.dup2 { old = outfd, new = Posix.FileSys.stderr } + ; Posix.Process.execp c) + handle OS.SysErr (err, _) => + ( print ("Fatal error in child: " ^ err ^ "\n") + ; OS.Process.exit OS.Process.failure )) + | SOME pid => (* Parent *) + let val _ = Posix.IO.close outfd + val (_, status) = Posix.Process.waitpid (Posix.Process.W_CHILD pid, []) + val output = readAllFDAsString (infd, 100) + val _ = Posix.IO.close infd + in (output, status) end + end + + +(* Check if a submitted file is a valid PDF using ghostscript. *) +fun checkPDF pdf = + let val spdf = joinHandinPath pdf in + case ( runCmd ("gs -o/dev/null -sDEVICE=nullpage " ^ spdf) + , Posix.FileSys.ST.size (Posix.FileSys.stat spdf) ) + of (_,0) => let val _ = printInBox [ "Warning: The empty file " ^ pdf ^ " is not a valid PDF document." + , "Please make sure to resubmit with a valid PDF document in its place." ] + in () end + | (0,_) => () + | _ => (abortWithMessage o stringsInBox) + [ "The file " ^ pdf ^ " is not a valid PDF document." + , "Please resubmit with a valid PDF document (or an empty file) in its place." + , "If you are convinced you submitted a valid PDF, please contact the course staff." ] + end + +(* Runs all of the checks and grades all of the problems in "checks". *) +fun runChecks (checks : checks list) = + List.foldl (fn (cs,results) => + case cs + of (Check (n, c)) => let val _ = printLn ("\n\nRunning check " ^ n ^ "...") + val _ = c () + val _ = printLn " Success.\n" + in results end + | (Problem (n, c)) => let val _ = printLn ("\n\nChecking problem " ^ n ^ "...") + val res = c () + val _ = printLn (" Score: " ^ (Real.toString res) ^ ".\n") + in (n, res) :: results end) + [] + checks + + +(* Returns a score of zero for all problems. *) +(* Useful when you need to abort but still provide a score. *) +fun failAll (checks : checks list) = + List.foldr (fn (cs,results) => + case cs + of Problem (n, c) => (n, 0) :: results + | _ => results) + [] + checks + +structure RE = RegExpFn(structure P = AwkSyntax + structure E = ThompsonEngine) + +fun matchesAwkRegex (r, s) = + let val r = RE.find (RE.compileString r) + in Option.isSome (StringCvt.scanString r s) end +end diff --git a/skel/main.sml b/skel/main.sml new file mode 100644 index 0000000..95e5bb5 --- /dev/null +++ b/skel/main.sml @@ -0,0 +1,25 @@ +(* Copyright (C) 2017 Ryan Kavanagh *) +(* Distributed under the ISC license, see COPYING for details. *) + +structure Main :> + sig + val main : (string * string list) -> OS.Process.status + end += +struct +structure H = Helper (structure C = struct val handinPath = "handin" end) +structure C = ChecksHelper(structure H = H) + +fun main _ = + let val scores = H.runChecks C.checks + val scoreboard = C.scoreboard scores + (* Make sure the scores are on the last line. *) + val _ = print "\n\n\n" + (* This below *must must must* be the last thing printed.. *) + val _ = H.printLn (H.scoresToString (scores, scoreboard)) + in OS.Process.success end + handle _ => (H.abortWithMessage o H.stringsInBox) + [ "Experienced uncaught exception!" + , "If you believe this to be in error, please contact your" + ^ " course staff." ] +end diff --git a/skel/run.sh b/skel/run.sh new file mode 100755 index 0000000..ff313b2 --- /dev/null +++ b/skel/run.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +abort () { + if [ "x$1" != "x" ]; then + echo "$1" + fi + printf "\n\n{\"scores\": {}}" + exit 0 +} + +tar -mxf autograde.tar || \ + abort "Failed to extract autograder." + +[ -d handin ] || mkdir handin + +[ -f handin.tar ] || \ + abort "Submission is expected to be at handin.tar. This file does not exist!" + +tar -xf handin.tar -C handin || \ + abort "Failed to extract submission." + +ml-build sources.cm Main.main grader || \ + abort "Failed to compile autograder. Please contact course staff." + +sml @SMLload grader.* || \ + abort "SML autograder exited with error. Please contact course staff." diff --git a/skel/sources.cm b/skel/sources.cm new file mode 100644 index 0000000..7359047 --- /dev/null +++ b/skel/sources.cm @@ -0,0 +1,9 @@ +Group is + $/basis.cm + $/regexp-lib.cm + support/sources.cm + helper.sig (* the HELPER signature *) + helper.sml (* the Helper module *) + checks.sig (* the CHECKS signature *) + checks.sml (* the ChecksHelper functor *) + main.sml (* the big cheese *) diff --git a/skel/support/README b/skel/support/README new file mode 100644 index 0000000..8c31e1a --- /dev/null +++ b/skel/support/README @@ -0,0 +1 @@ +Put any support files here. diff --git a/skel/support/sources.cm b/skel/support/sources.cm new file mode 100644 index 0000000..fad8872 --- /dev/null +++ b/skel/support/sources.cm @@ -0,0 +1,2 @@ +(* Add any extra support SML files here *) +Group is diff --git a/skel/test_checks.sh b/skel/test_checks.sh new file mode 100755 index 0000000..975ea4c --- /dev/null +++ b/skel/test_checks.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +make -s autograde + +for f in test_handins/*.tar; do + base=`basename -s .tar ${f}` + tmp=`mktemp -d` + cp ../autograde.tar ${tmp} + cp test_handins/${base}.tar ${tmp}/handin.tar + cp ../autograde-Makefile ${tmp} + echo "\e[41m!!!!!!! Last 10 output lines for test ${base}:\e[0m" + (cd ${tmp}; make -s -f autograde-Makefile | tail -n 10) + echo "\e[42m!!!!!!! Expected result:\e[0m" + cat test_handins/${base}.exp + rm -fr ${tmp} +done diff --git a/skel/test_handins/README b/skel/test_handins/README new file mode 100644 index 0000000..b43561b --- /dev/null +++ b/skel/test_handins/README @@ -0,0 +1,5 @@ +To test your checks.sml, place sample submissions in this directory +according to the following format: + + somename.tar : the submission you want to test + somename.exp : the expected json score -- cgit v1.2.3