diff --git a/nselib/data/drupal-modules.lst b/nselib/data/drupal-modules.lst
index 47b8631bf..0bd11dc80 100644
--- a/nselib/data/drupal-modules.lst
+++ b/nselib/data/drupal-modules.lst
@@ -598,7 +598,6 @@ imagefield_focus
 ocupload
 paragraphs
 node_import
-node_import
 manualcrop
 image_link_formatter
 subpathauto
@@ -3398,7 +3397,6 @@ services_tools
 followbutton
 radix_core
 tb_social_feed
-tb_social_feed
 biblio_scholar
 commerce_pado
 lowername
@@ -3673,7 +3671,6 @@ ctr
 og_forum
 autotagging
 iframe_page
-iframe_page
 taxonomy_tree_select
 safer_login
 commons_release
@@ -3723,7 +3720,6 @@ filedepot_linkit
 media_oembed_thumbnail_style
 translated_entityreference
 views_accelerator
-views_accelerator
 gd_infinite_scroll
 cms_contact
 cms_portfolio
@@ -3773,7 +3769,6 @@ topichubs
 ooyala
 content_access_view
 video_upload
-video_upload
 gmapez
 password_change
 sms_clickatell
@@ -3823,7 +3818,6 @@ commerce_eu_vat
 new_relic_rpm
 mail_redirect
 views_chart_tools
-views_chart_tools
 removeaddnewcommentteaserlink
 userpoints_votingapi
 user_selectable_roles
@@ -12998,7 +12992,6 @@ macrotags
 form_mail
 flexinode
 navigation
-navigation
 jsdomenu
 importpage
 waypath
@@ -13048,7 +13041,6 @@ editasnew
 nodeperm_role
 helpedit
 powells
-powells
 pureftp
 peoplesemailnetwork
 challenge_response
@@ -13173,7 +13165,6 @@ drupman
 gdata
 pledgebank
 karma
-karma
 simpletestauto
 lightcrm
 taxonomy_ezfilter
@@ -13348,7 +13339,6 @@ pressflow_multidomain
 rpg
 gmap_civigroup
 globalnode
-globalnode
 search_attachments
 og2mlm
 crm
@@ -13548,7 +13538,6 @@ ventutils
 publisher
 pricegun
 daemon
-daemon
 import_contacts
 natsort
 recycle_node
@@ -13623,7 +13612,6 @@ sso_multidb
 activeedit
 adbard
 savetofile
-savetofile
 subscriptions_author
 jquery_blockui
 taxtractor
@@ -13748,7 +13736,6 @@ urlproxy
 webform_pdf
 nodetaxonomy_notifications
 title_perms
-title_perms
 coppa
 profile_map
 uc_cart_theme
@@ -13848,7 +13835,6 @@ bandwidth
 roundcube
 draggable_blocks
 userprofile
-userprofile
 roboconf
 tbp
 srt
@@ -13898,7 +13884,6 @@ createcontent
 generic_timesheet
 ec_useracc
 citationcounts
-citationcounts
 iphone
 rdl
 pbs
@@ -13948,7 +13933,6 @@ sopac
 changelogreader
 sharepal
 patch
-patch
 dataimport
 geo_gui
 contribute
@@ -14073,7 +14057,6 @@ micropayments
 ernest_marples
 amazon_related
 media_rec
-media_rec
 ootools
 release
 shortlink
@@ -14123,7 +14106,6 @@ domain_theme
 domain_user
 ad_blockers_detector
 italianstemmer
-italianstemmer
 pcapbt
 hydra_network
 hydra_slideshow
@@ -14173,7 +14155,6 @@ context_menu
 ccb
 acs
 imagecache_utils
-imagecache_utils
 visual_admin
 i18n_auto_draft
 evalapi
@@ -14348,7 +14329,6 @@ apachesolr_multiserver
 mobwrite
 freebase
 content_import
-content_import
 uc_icici
 cardgame
 hs_subgroups
@@ -14573,7 +14553,6 @@ open311
 output
 multiquote
 uc_receipt
-uc_receipt
 vf_vinquery
 vf_dataone
 bartik_bonus
@@ -14673,7 +14652,6 @@ taxpages
 group_space_url
 surveys_points
 surveys_newsletter
-surveys_newsletter
 xgitsha
 appinsight
 campaign_monitor_integrated
@@ -14773,7 +14751,6 @@ smskit
 registry
 mailwire
 term_status
-term_status
 saasu
 backup_updated_files
 websocket
@@ -14823,7 +14800,6 @@ google_picker
 sphinxapi
 eclipse_code_validator
 discursive_features
-discursive_features
 drupal_ruble
 user_status_color
 context_auto
@@ -14998,7 +14974,6 @@ pid
 whami_source_adapter
 kill_blocks
 drush_clone
-drush_clone
 plugin_stub
 nupoll
 arrow
@@ -15098,7 +15073,6 @@ i18n_views_fbcl
 hook_hunter
 project_estimator
 make_hg
-make_hg
 voipmake
 spamrecord
 dynamic
@@ -15223,7 +15197,6 @@ broken_tests
 audit_trail
 commerce_securepay
 domain_country
-domain_country
 media_mpx
 explorer
 provision_profile_settings
@@ -15273,7 +15246,6 @@ d64_lsr_stopper
 japanese_era
 imagetobase64calc
 geodb
-geodb
 movico
 premium_fields
 region
@@ -15598,7 +15570,6 @@ matching_gift
 ajaxchat
 webform_form_element
 commerce_qmbs
-commerce_qmbs
 unicorns
 air
 konzilo
@@ -15873,7 +15844,6 @@ scss
 hosting_webhooks
 ibpcatalog
 deploy_check
-deploy_check
 erus
 field_formatter
 ip2locale_geoip
@@ -15973,7 +15943,6 @@ voipusernumber
 image_plus
 block_aria_label
 reddit_api
-reddit_api
 taxonomy_term_page_access
 commerce_shipping_cd
 voippoll
@@ -16223,7 +16192,6 @@ commerce_winbiz
 xpathparser
 gentleselect
 zoetrope_viewer
-zoetrope_viewer
 cue_field
 config_refresh
 niftycorners_module
@@ -16298,7 +16266,6 @@ mobile_menu
 hotel_california
 scald_iframe
 pay_block
-pay_block
 pay_ipizza
 master_ui
 drush_nagios
@@ -16673,7 +16640,6 @@ southafrican_id
 nestedbox
 drupalvm
 parish_info
-parish_info
 facetapi_merge_options
 piecentage
 sql_views
@@ -16748,7 +16714,6 @@ priority_queue
 element
 localefix
 taarikh
-taarikh
 html_diff
 drupalextension_remote
 register_with_stripe
@@ -16948,7 +16913,6 @@ janrain
 jquery_oembed_all
 parser_salesforce
 context_profile_role
-context_profile_role
 lightlazyloader
 fieldupdate
 stripe_checkout
@@ -17173,7 +17137,6 @@ imce_private_files
 ief_dialog
 plug
 asf
-asf
 mixcloud_field
 video_embed_instagram
 modules_collapse
diff --git a/nselib/data/http-folders.txt b/nselib/data/http-folders.txt
index 996cc4898..07d395ac1 100644
--- a/nselib/data/http-folders.txt
+++ b/nselib/data/http-folders.txt
@@ -21,7 +21,6 @@
 /admcgi/
 /admentor/
 /admin/
-/admin/
 /admin_/
 /admin.back/
 /admin-bak/
@@ -55,7 +54,6 @@
 /appsec/
 /ar/
 /archive/
-/archive/
 /archives/
 /arcsight/
 /asa/
@@ -74,11 +72,9 @@
 /back/
 /backend/
 /backup/
-/backup/
 /backups/
 /bad/
 /bak/
-/bak/
 /banca/
 /banco/
 /bank/
@@ -93,11 +89,8 @@
 /bdatos/
 /beef/
 /beta/
-/beta/
 /billpay/
 /bin/
-/bin/
-/bin/
 /binaries/
 /binary/
 /blog/
@@ -138,14 +131,10 @@
 /cfdocs/
 /cfide/
 /cgi/
-/cgi/
 /cgi-914/
 /cgi-915/
 /cgi-auth/
 /cgibin/
-/cgibin/
-/cgi-bin/
-/cgi-bin/
 /cgi-bin/
 /cgi-bin2/
 /cgi.cgi/
@@ -155,24 +144,19 @@
 /cgilib/
 /cgi-lib/
 /cgi-local/
-/cgi-local/
 /cgi-perl/
 /cgis/
-/cgis/
 /cgiscripts/
 /cgi-scripts/
 /cgi-shl/
 /cgi-shop/
 /cgi-sys/
-/cgi-sys/
 /cgi-weddico/
 /cgiwin/
 /cgi-win/
-/cgi-win/
 /Citrix/
 /class/
 /classes/
-/classes/
 /client/
 /cliente/
 /clientes/
@@ -193,7 +177,6 @@
 /conecta/
 /conf/
 /config/
-/config/
 /configs/
 /configure/
 /connect/
@@ -214,7 +197,6 @@
 /CS/
 /csr/
 /css/
-/css/
 /cuenta/
 /cuentas/
 /currency/
@@ -229,14 +211,12 @@
 /darkportal/
 /dat/
 /data/
-/data/
 /database/
 /databases/
 /datafiles/
 /dato/
 /datos/
 /db/
-/db/
 /dbase/
 /dcforum/
 /ddreport/
@@ -247,7 +227,6 @@
 /delete/
 /deleted/
 /demo/
-/demo/
 /demoauct/
 /demomall/
 /demos/
@@ -256,7 +235,6 @@
 /derived/
 /design/
 /dev/
-/dev/
 /devel/
 /development/
 /dir/
@@ -283,7 +261,6 @@
 /down/
 /download/
 /downloads/
-/downloads/
 /dump/
 /durep/
 /e/
@@ -322,7 +299,6 @@
 /fbsd/
 /fcgi/
 /fcgi-bin/
-/fcgi-bin/
 /features/
 /file/
 /fileadmin/
@@ -338,9 +314,6 @@
 /formsmgr/
 /form-totaller/
 /forum/
-/forum/
-/forum/
-/forums/
 /forums/
 /foto/
 /fotos/
@@ -390,7 +363,6 @@
 /hp/
 /ht/
 /htbin/
-/htbin/
 /htdocs/
 /htm/
 /html/
@@ -403,17 +375,14 @@
 /ibill/
 /IBMWebAS/
 /icons/
-/icons/
 /idea/
 /ideas/
 /iisadmin/
 /iissamples/
-/iissamples/
 /image/
 /imagenes/
 /imagery/
 /images/
-/images/
 /img/
 /imp/
 /import/
@@ -422,8 +391,6 @@
 /inc/
 /include/
 /includes/
-/includes/
-/incoming/
 /incoming/
 /index/
 /inet/
@@ -433,12 +400,9 @@
 /ingresa/
 /ingreso/
 /install/
-/install/
 /internal/
 /internet/
 /intranet/
-/intranet/
-/intranet/
 /inventory/
 /invitado/
 /isapi/
@@ -486,18 +450,14 @@
 /logfiles/
 /logg/
 /logger/
-/logger/
 /logging/
 /login/
-/login/
 /logon/
 /logout/
 /logs/
-/logs/
 /lost+found/
 /m/
 /mail/
-/mail/
 /mail_log_files/
 /mailman/
 /mailroot/
@@ -508,15 +468,12 @@
 /management/
 /manager/
 /manual/
-/manual/
 /map/
 /maps/
 /marketing/
 /mediawiki/
 /mem/
 /member/
-/member/
-/members/
 /members/
 /mem_bin/
 /message/
@@ -528,7 +485,6 @@
 /mod/
 /module/
 /modules/
-/modules/
 /movimientos/
 /mpcgi/
 /mqseries/
@@ -558,7 +514,6 @@
 /netshare/
 /nettracker/
 /network/
-/network/
 /new/
 /news/
 /News/
@@ -608,14 +563,10 @@
 /phorum/
 /php/
 /phpBB/
-/phpBB/
 /php_classes/
 /phpclassifieds/
 /phpimageview/
 /phpmyadmin/
-/phpmyadmin/
-/phpMyAdmin/
-/phpMyAdmin/
 /phpMyAdmin/
 /phpnuke/
 /phpPhotoAlbum/
@@ -626,7 +577,6 @@
 /pike/
 /piranha/
 /pls/
-/pls/
 /plsql/
 /plssample/
 /plssampleadmin/
@@ -646,19 +596,15 @@
 /privacy/
 /privado/
 /private/
-/private/
 /_private/
 /prod/
 /projectserver/
 /protected/
-/protected/
 /proxy/
 /prueba/
 /pruebas/
 /prv/
 /pub/
-/pub/
-/public/
 /public/
 /_public/
 /publica/
@@ -684,7 +630,6 @@
 /reports/
 /reseller/
 /restricted/
-/restricted/
 /retail/
 /reveal/
 /reviews/
@@ -703,13 +648,11 @@
 /script/
 /ScriptLibrary/
 /scripts/
-/scripts/
 /search/
 /search-ui/
 /sec/
 /secret/
 /secure/
-/secure/
 /secured/
 /security/
 /sell/
@@ -805,7 +748,6 @@
 /templates/
 /temporal/
 /test/
-/test/
 /test-cgi/
 /testing/
 /tests/
@@ -816,7 +758,6 @@
 /tip/
 /tips/
 /tmp/
-/tmp/
 /ToDo/
 /tool/
 /tools/
@@ -879,7 +820,6 @@
 /web800fo/
 /webaccess/
 /webadmin/
-/webadmin/
 /webAdmin/
 /webalizer/
 /webapps/