Merge pull request #173 from Puyodead1/feat/quizes

Add support for quizzes

main.py

@@ -33,8 +33,9 @@ cookies = ""
 downloader = None
 logger: logging.Logger = None
 dl_assets = False
-skip_lectures = False
 dl_captions = False
+dl_quizzes = False
+skip_lectures = False
 caption_locale = "en"
 quality = None
 bearer_token = None
@@ -67,7 +68,7 @@ def log_subprocess_output(prefix: str, pipe: IO[bytes]):
 
 # this is the first function that is called, we parse the arguments, setup the logger, and ensure that required directories exist
 def pre_run():
-    global cookies, dl_assets, skip_lectures, dl_captions, caption_locale, quality, bearer_token, portal_name, course_name, keep_vtt, skip_hls, concurrent_downloads, disable_ipv6, load_from_file, save_to_file, bearer_token, course_url, info, logger, keys, id_as_course_name, is_subscription_course, LOG_LEVEL, use_h265, h265_crf, h265_preset, use_nvenc
+    global cookies, dl_assets, dl_captions, dl_quizzes, skip_lectures, caption_locale, quality, bearer_token, course_name, keep_vtt, skip_hls, concurrent_downloads, disable_ipv6, load_from_file, save_to_file, bearer_token, course_url, info, logger, keys, id_as_course_name, is_subscription_course, LOG_LEVEL, use_h265, h265_crf, h265_preset, use_nvenc
 
     # make sure the directory exists
     if not os.path.exists(DOWNLOAD_DIR):
@@ -131,6 +132,12 @@ def pre_run():
         action="store_true",
         help="If specified, captions will be downloaded",
     )
+    parser.add_argument(
+        "--download-quizzes",
+        dest="download_quizzes",
+        action="store_true",
+        help="If specified, quizzes will be downloaded",
+    )
     parser.add_argument(
         "--keep-vtt",
         dest="keep_vtt",
@@ -215,6 +222,8 @@ def pre_run():
         caption_locale = args.lang
     if args.download_captions:
         dl_captions = True
+    if args.download_quizzes:
+        dl_quizzes = True
     if args.skip_lectures:
         skip_lectures = True
     if args.quality:
@@ -333,6 +342,25 @@ class Udemy:
             logger.fatal("Login Failure! You are probably missing an access token!")
             sys.exit(1)
 
+    def _get_quiz(self, quiz_id):
+        print(portal_name)
+        self.session._headers.update(
+            {
+                "Host": "{portal_name}.udemy.com".format(portal_name=portal_name),
+                "Referer": "https://{portal_name}.udemy.com/course/{course_name}/learn/quiz/{quiz_id}".format(portal_name=portal_name, course_name=course_name, quiz_id=quiz_id),
+            }
+        )
+        url = QUIZ_URL.format(portal_name=portal_name, quiz_id=quiz_id)
+        print(url)
+        try:
+            resp = self.session._get(url).json()
+        except conn_error as error:
+            logger.fatal(f"[-] Udemy Says: Connection error, {error}")
+            time.sleep(0.8)
+            sys.exit(1)
+        else:
+            return resp.get("results")
+
     def _extract_supplementary_assets(self, supp_assets, lecture_counter):
         _temp = []
         for entry in supp_assets:
@@ -648,7 +676,7 @@ class Udemy:
             results = webpage.get("results", [])
         return results
 
-    def _extract_course_info_json(self, url, course_id, portal_name):
+    def _extract_course_info_json(self, url, course_id):
         self.session._headers.update({"Referer": url})
         url = COURSE_INFO_URL.format(portal_name=portal_name, course_id=course_id)
         try:
@@ -836,12 +864,14 @@ class Udemy:
         data_args = data.attrs["data-module-args"]
         data_json = json.loads(data_args)
         course_id = data_json.get("courseId", None)
-        portal_name = self.extract_portal_name(url)
+        return course_id
-        return course_id, portal_name
 
     def _extract_course_info(self, url):
+        global portal_name
         portal_name, course_name = self.extract_course_name(url)
-        course = {}
+        course = {
+            "portal_name": portal_name
+        }
 
         if not is_subscription_course:
             results = self._subscribed_courses(portal_name=portal_name, course_name=course_name)
@@ -857,11 +887,10 @@ class Udemy:
                 course = self._extract_course(response=results, course_name=course_name)
 
         if not course or is_subscription_course:
-            course_id, portal_name = self._extract_subscription_course_info(url)
+            course_id = self._extract_subscription_course_info(url)
-            course = self._extract_course_info_json(url, course_id, portal_name)
+            course = self._extract_course_info_json(url, course_id)
 
         if course:
-            course.update({"portal_name": portal_name})
             return course.get("id"), course
         if not course:
             logger.fatal("Downloading course information, course id not found .. ")
@@ -1545,6 +1574,27 @@ def process_lecture(lecture, lecture_path, lecture_file_name, chapter_dir):
             logger.error("      > Missing sources for lecture", lecture)
 
 
+
+def process_quiz(udemy: Udemy, lecture, chapter_dir):
+    lecture_title = lecture.get("lecture_title")
+    lecture_index = lecture.get("lecture_index")
+    lecture_file_name = sanitize_filename(lecture_title + ".html")
+    lecture_path = os.path.join(chapter_dir, lecture_file_name)
+
+    logger.info(f"  > Processing quiz {lecture_index}")
+    questions = udemy._get_quiz(lecture.get("id"))
+    with open("quiz_template.html", "r") as f:
+        html = f.read()
+        quiz_data = {
+            "pass_percent": lecture.get("data").get("pass_percent"),
+            "questions": questions,
+        }
+        html = html.replace("__data_placeholder__", json.dumps(quiz_data))
+        with open(lecture_path, "w") as f:
+            f.write(html)
+
+
+
 def parse_new(udemy: Udemy, udemy_object: dict):
     total_chapters = udemy_object.get("total_chapters")
     total_lectures = udemy_object.get("total_lectures")
@@ -1565,10 +1615,16 @@ def parse_new(udemy: Udemy, udemy_object: dict):
         logger.info(f"======= Processing chapter {chapter_index} of {total_chapters} =======")
 
         for lecture in chapter.get("lectures"):
-            index = lecture.get("index")  # this is lecture_counter
+            clazz = lecture.get("_class")
-            lecture_index = lecture.get("lecture_index")  # this is the raw object index from udemy
-            lecture_title = lecture.get("lecture_title")
 
+            if clazz == "quiz" and dl_quizzes:
+                process_quiz(udemy, lecture, chapter_dir)
+                continue
+
+            index = lecture.get("index")  # this is lecture_counter
+            # lecture_index = lecture.get("lecture_index")  # this is the raw object index from udemy
+
+            lecture_title = lecture.get("lecture_title")
             parsed_lecture = udemy._parse_lecture(lecture)
 
             lecture_extension = parsed_lecture.get("extension")
@@ -1579,8 +1635,9 @@ def parse_new(udemy: Udemy, udemy_object: dict):
             lecture_file_name = sanitize_filename(lecture_title + "." + extension)
             lecture_path = os.path.join(chapter_dir, lecture_file_name)
 
-            logger.info(f"  > Processing lecture {lecture_index} of {total_lectures}")
             if not skip_lectures:
+                logger.info(f"  > Processing lecture {index} of {total_lectures}")
+
                 # Check if the lecture is already downloaded
                 if os.path.isfile(lecture_path):
                     logger.info("      > Lecture '%s' is already downloaded, skipping..." % lecture_title)
@@ -1722,7 +1779,7 @@ def _print_course_info(udemy: Udemy, udemy_object: dict):
 
 
 def main():
-    global bearer_token
+    global bearer_token, portal_name
     aria_ret_val = check_for_aria()
     if not aria_ret_val:
         logger.fatal("> Aria2c is missing from your system or path!")
@@ -1758,7 +1815,6 @@ def main():
         if course_info and isinstance(course_info, dict):
             title = sanitize_filename(course_info.get("title"))
             course_title = course_info.get("published_title")
-            portal_name = course_info.get("portal_name")
 
     logger.info("> Fetching course content, this may take a minute...")
     if load_from_file:
@@ -1768,6 +1824,8 @@ def main():
         portal_name = course_json.get("portal_name")
     else:
         course_json = udemy._extract_course_json(course_url, course_id, portal_name)
+        course_json["portal_name"] = portal_name
+
     if save_to_file:
         with open(os.path.join(os.getcwd(), "saved", "course_content.json"), encoding="utf8", mode="w") as f:
             f.write(json.dumps(course_json))
@@ -1815,13 +1873,13 @@ def main():
                 elif clazz == "lecture":
                     lecture_counter += 1
                     lecture_id = entry.get("id")
-                    if len(udemy_object["chapters"]) == 0:
+                    # if len(udemy_object["chapters"]) == 0:
-                        lectures = []
+                    #     lectures = []
-                        chapter_index = entry.get("object_index")
+                    #     chapter_index = entry.get("object_index")
-                        chapter_title = "{0:02d} - ".format(chapter_index) + sanitize_filename(entry.get("title"))
+                    #     chapter_title = "{0:02d} - ".format(chapter_index) + sanitize_filename(entry.get("title"))
-                        if chapter_title not in udemy_object["chapters"]:
+                    #     if chapter_title not in udemy_object["chapters"]:
-                            udemy_object["chapters"].append({"chapter_title": chapter_title, "chapter_id": lecture_id, "chapter_index": chapter_index, "lectures": []})
+                    #         udemy_object["chapters"].append({"chapter_title": chapter_title, "chapter_id": lecture_id, "chapter_index": chapter_index, "lectures": []})
-                            counter += 1
+                    #         counter += 1
 
                     if lecture_id:
                         logger.info(f"Processing {course.index(entry)} of {len(course)}")
@@ -1829,26 +1887,35 @@ def main():
                         lecture_index = entry.get("object_index")
                         lecture_title = "{0:03d} ".format(lecture_counter) + sanitize_filename(entry.get("title"))
 
-                        lectures.append({"index": lecture_counter, "lecture_index": lecture_index, "lecture_title": lecture_title, "data": entry})
+                        lectures.append({"index": lecture_counter, "lecture_index": lecture_index, "lecture_title": lecture_title, "_class": entry.get("_class"), "id": lecture_id, "data": entry})
                     udemy_object["chapters"][counter]["lectures"] = lectures
                     udemy_object["chapters"][counter]["lecture_count"] = len(lectures)
                 elif clazz == "quiz":
+                    lecture_counter += 1
                     lecture_id = entry.get("id")
-                    if len(udemy_object["chapters"]) == 0:
+                    # if len(udemy_object["chapters"]) == 0:
-                        lectures = []
+                    #     lectures = []
-                        chapter_index = entry.get("object_index")
+                    #     chapter_index = entry.get("object_index")
-                        chapter_title = "{0:02d} - ".format(chapter_index) + sanitize_filename(entry.get("title"))
+                    #     chapter_title = "{0:02d} - ".format(chapter_index) + sanitize_filename(entry.get("title"))
-                        if chapter_title not in udemy_object["chapters"]:
+                    #     if chapter_title not in udemy_object["chapters"]:
-                            lecture_counter = 0
+                    #         lecture_counter = 0
-                            udemy_object["chapters"].append(
+                    #         udemy_object["chapters"].append(
-                                {
+                    #             {
-                                    "chapter_title": chapter_title,
+                    #                 "chapter_title": chapter_title,
-                                    "chapter_id": lecture_id,
+                    #                 "chapter_id": lecture_id,
-                                    "chapter_index": chapter_index,
+                    #                 "chapter_index": chapter_index,
-                                    "lectures": [],
+                    #                 "lectures": [],
-                                }
+                    #             }
-                            )
+                    #         )
-                            counter += 1
+                    #         counter += 1
+
+                    if lecture_id:
+                        logger.info(f"Processing {course.index(entry)} of {len(course)}")
+
+                        lecture_index = entry.get("object_index")
+                        lecture_title = "{0:03d} ".format(lecture_counter) + sanitize_filename(entry.get("title"))
+
+                        lectures.append({"index": lecture_counter, "lecture_index": lecture_index, "lecture_title": lecture_title, "_class": entry.get("_class"), "id": lecture_id, "data": entry})
 
                     udemy_object["chapters"][counter]["lectures"] = lectures
                     udemy_object["chapters"][counter]["lectures_count"] = len(lectures)
@@ -1860,6 +1927,7 @@ def main():
             with open(os.path.join(os.getcwd(), "saved", "_udemy.json"), encoding="utf8", mode="w") as f:
                 # remove "bearer_token" from the object before writing
                 udemy_object.pop("bearer_token")
+                udemy_object["portal_name"] = portal_name
                 f.write(json.dumps(udemy_object))
                 f.close()
             logger.info("> Saved parsed data to json")

quiz_template.html

@@ -0,0 +1,263 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Quiz</title>
+
+    <style>
+      body {
+        font-family: sf pro text, -apple-system, BlinkMacSystemFont, Roboto,
+          segoe ui, Helvetica, Arial, sans-serif, apple color emoji,
+          segoe ui emoji, segoe ui symbol;
+        font-weight: 400;
+        line-height: 22.4px;
+        font-size: 16px;
+      }
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      ul,
+      ol {
+        font-size: 16px;
+        font-weight: 400;
+      }
+      ul {
+        list-style: none;
+        margin: 0;
+        padding: 0;
+        max-width: none;
+      }
+      code {
+        background-color: #fff;
+        border: 1px solid #d1d7dc;
+        color: #b4690e;
+        font-size: 90%;
+        padding: 0.2rem 0.4rem;
+      }
+      .quiz-content {
+        padding: 2.4rem;
+        word-break: break-word;
+        max-width: 86rem;
+        margin: 0 auto;
+      }
+      .quiz-container {
+        margin: 0 auto;
+        max-width: 84rem;
+        padding: 0;
+      }
+      .question {
+        margin-bottom: 5rem;
+      }
+      .question span {
+        font-size: 24px;
+        font-weight: 700;
+      }
+      .question-prompt {
+        margin-top: 0.8rem;
+        font-weight: 700;
+      }
+      .question-answer {
+        margin-top: 1.6rem;
+        padding-left: 0;
+        box-sizing: border-box;
+      }
+      .question-answer label {
+        max-width: 80rem;
+        position: relative;
+        cursor: pointer;
+        display: flex;
+        min-width: 18rem;
+        border: solid #1c1d1f 2px;
+      }
+      .question-answer div {
+        display: flex;
+        align-items: center;
+        padding: 0.5rem;
+        width: 100%;
+      }
+      .question-answer div:hover {
+        background-color: #f5f5f5;
+      }
+      .question-answer input {
+        display: none;
+      }
+      .question-answer span {
+        width: 1rem;
+        height: 1rem;
+        margin-right: 1.6rem;
+        top: 0;
+        border-radius: 50%;
+        display: inline-block;
+        flex-shrink: 0;
+        border: 0.2rem solid #1c1d1f;
+        position: relative;
+      }
+      .selected {
+        background: #1c1d1f;
+        box-shadow: 0 0 0 0.4rem #fff inset;
+      }
+      .score {
+              position: fixed;
+            }
+    </style>
+  </head>
+
+  <body onload="main()">
+    <div id="score" class="score">
+      <span>Score: N/A of N/A</span>
+    </div>
+    <div id="quiz-container" class="quiz-content"></div>
+
+    <script>
+      const quizData = __data_placeholder__;
+      var correct = 0;
+      var total = 0;
+      const questionData = quizData.questions
+      const passPercent = quizData.pass_percent
+
+      function main() {
+        total = questionData.length;
+
+        var questions = [];
+        for (var i = 0; i < questionData.length; i++) {
+          var question = questionData[i];
+          var questionText = question.prompt.question;
+          var answers = question.prompt.answers;
+          var correctAnswer = question.correct_response[0];
+          var correctAnswerText = answers[correctAnswer.charCodeAt(0) - 97];
+          var questionObj = {
+            question: questionText,
+            correctAnswer: correctAnswerText,
+            answers: answers,
+            id: question.id,
+          };
+          questions.push(questionObj);
+        }
+
+        updateScore();
+
+        // display the questions
+        var questionsContainer = document.getElementById("quiz-container");
+        for (var i = 0; i < questions.length; i++) {
+          var question = questions[i];
+          var questionElement = document.createElement("form");
+          questionElement.className = "question";
+          questionElement.innerHTML =
+            "<span>Question " +
+            (i + 1) +
+            ":</span>" +
+            '<div class="question-prompt">' +
+            question.question +
+            "</div>";
+          questionElement.id = question.id;
+          questionElement.classList.add("quiz-container");
+          var answersElement = document.createElement("ul");
+          answersElement.className = "answers";
+          for (var j = 0; j < question.answers.length; j++) {
+            var answer = question.answers[j];
+            var answerElement = document.createElement("li");
+            answerElement.className = "answer";
+            answerElement.innerHTML =
+              '<label for="input-' +
+              question.id +
+              "-" +
+              j +
+              '"><div onclick="select(' +
+              question.id +
+              ", " +
+              j +
+              ')"><span></span><input id="input-' +
+              question.id +
+              "-" +
+              j +
+              '" type="radio" name="question' +
+              i +
+              '" value="' +
+              j +
+              '">' +
+              answer +
+              "</input></div></label>";
+            answerElement.classList.add("question-answer");
+            answersElement.appendChild(answerElement);
+          }
+
+          questionElement.appendChild(answersElement);
+
+          // add a submit button
+          var submitButton = document.createElement("button");
+          submitButton.className = "submit";
+          submitButton.innerHTML = "Submit";
+          submitButton.classList.add("submit-button");
+
+          submitButton.addEventListener("click", function (e) {
+            e.preventDefault();
+            var questionElement = e.target.parentElement;
+            var questionId = questionElement.id;
+            var question = questions.filter(function (q) {
+              return q.id == questionId;
+            })[0];
+
+            // set the border color of all answers to black
+            var answers = questionElement.getElementsByClassName("answer");
+            for (var i = 0; i < answers.length; i++) {
+              console.log(answers[i]);
+              answers[i].children[0].style.borderColor = "#1c1d1f";
+            }
+
+            var answer = questionElement.querySelector(
+              'input[type="radio"]:checked'
+            );
+            if (answer) {
+              var answerIndex = answer.value;
+              var answerText = question.answers[answerIndex];
+              if (answerText == question.correctAnswer) {
+                answer.parentElement.parentElement.style.borderColor =
+                  "limegreen";
+                // alert("Correct!");
+                correct++;
+                updateScore();
+              } else {
+                answer.parentElement.parentElement.style.borderColor = "red";
+                // alert("Incorrect!");
+              }
+            } else {
+              alert("Please select an answer.");
+            }
+          });
+
+          // add button
+          questionElement.appendChild(submitButton);
+
+          questionsContainer.appendChild(questionElement);
+        }
+      }
+
+      function select(question, answer) {
+        var questionElement = document.getElementById(question);
+        var answers = questionElement.getElementsByClassName("answer");
+        for (var i = 0; i < answers.length; i++) {
+          var span = answers[i].children[0].children[0].children[0];
+          var input = answers[i].children[0].children[0].children[1];
+          span.classList.remove("selected");
+          input.checked = false;
+        }
+        var span = answers[answer].children[0].children[0].children[0];
+        var input = answers[answer].children[0].children[0].children[1];
+        span.classList.add("selected");
+        input.checked = true;
+      }
+
+      function updateScore() {
+        var scoreElem = document.getElementById("score");
+        const score = (correct / total) * 100;
+        scoreElem.innerHTML = "Score: " + score + "/" + passPercent + "%";
+      }
+    </script>
+  </body>
+</html>