Compare commits
No commits in common. "58ef806e56d5d42d504e5e776333e989ffd9dc8f" and "e469f7593a60e87bc592ee53b0e8573583da6d08" have entirely different histories.
58ef806e56
...
e469f7593a
|
@ -5,8 +5,8 @@ Goal - a commandline-based tool for creating blocklists for ActivityPub software
|
||||||
## Planned features
|
## Planned features
|
||||||
|
|
||||||
- [ ] Accept and parse commandline arguments
|
- [ ] Accept and parse commandline arguments
|
||||||
- [ ] Create tiered or thresholded lists
|
- [ ] Add ability to specify reasons
|
||||||
- [ ] Support CSV and JSON
|
|
||||||
- [ ] Use URLs as sources
|
- [ ] Use URLs as sources
|
||||||
- [ ] Directly request lists from API endpoints
|
- [ ] Directly request lists from API endpoints
|
||||||
- [ ] Add ability to specify reasons
|
- [ ] Create tiered or thresholded lists
|
||||||
|
- [ ] Support CSV and JSON
|
13
src/main.rs
13
src/main.rs
|
@ -22,10 +22,6 @@ struct Cli {
|
||||||
#[arg(short = 'M', long)]
|
#[arg(short = 'M', long)]
|
||||||
mute: Vec<PathBuf>,
|
mute: Vec<PathBuf>,
|
||||||
|
|
||||||
/// Specifies a source (*.block.txt, *.mute.txt)
|
|
||||||
#[arg(short = 'S', long)]
|
|
||||||
src: Vec<PathBuf>,
|
|
||||||
|
|
||||||
/// Specifies confidence in a source. Default = 100
|
/// Specifies confidence in a source. Default = 100
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
trust: Vec<u16>,
|
trust: Vec<u16>,
|
||||||
|
@ -38,17 +34,14 @@ struct Cli {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
env_logger::init(); // TODO add more logging
|
env_logger::init();
|
||||||
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
if cli.block.is_empty() && cli.mute.is_empty() && cli.config.is_none() {
|
if cli.block.is_empty() && cli.mute.is_empty() && cli.config.is_none() {
|
||||||
error!("No lists or configuration provided.");
|
error!("No lists or configuration provided.");
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO parse config file if one is provided
|
|
||||||
|
|
||||||
// TODO read modsources from files
|
// TODO argument parsing - IN PROGRESS
|
||||||
// TODO combine modsources into modmap
|
// TODO logging
|
||||||
// TODO write modmap to files
|
|
||||||
}
|
}
|
||||||
|
|
82
src/manip.rs
82
src/manip.rs
|
@ -4,35 +4,35 @@ use log::error;
|
||||||
|
|
||||||
// ENUMS
|
// ENUMS
|
||||||
|
|
||||||
/// Specifies a limit type, whether block or silence
|
/// Specifies a moderation action, whether block or silence
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub enum Limit {
|
pub enum ModAction {
|
||||||
Block,
|
Block,
|
||||||
Silence,
|
Silence,
|
||||||
}
|
}
|
||||||
|
|
||||||
// STRUCTS
|
// STRUCTS
|
||||||
|
|
||||||
/// Wraps a host's weighted limits
|
/// Indicates weight of moderation action on a host
|
||||||
#[derive(Debug, Default, Eq, PartialEq)]
|
#[derive(Debug, Default, Eq, PartialEq)]
|
||||||
pub struct LimitIndices {
|
pub struct ModActionTrust {
|
||||||
pub block: u16,
|
pub block: u16,
|
||||||
pub silence: u16,
|
pub silence: u16,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LimitIndices {
|
impl ModActionTrust {
|
||||||
pub fn add_limit(&mut self, limit: Limit, weight: u16) -> &mut Self {
|
pub fn add_action(&mut self, action: ModAction, trust: u16) -> &mut Self {
|
||||||
match limit {
|
match action {
|
||||||
Limit::Block => self.block += weight,
|
ModAction::Block => self.block += trust,
|
||||||
Limit::Silence => self.silence += weight,
|
ModAction::Silence => self.silence += trust,
|
||||||
}
|
}
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<(u16, u16)> for LimitIndices {
|
impl From<(u16, u16)> for ModActionTrust {
|
||||||
/// Creates indices from a tuple of two `u16` weights. Useful mostly for
|
/// Creates mod action weights from a tuple of two `u16` weights. Useful
|
||||||
/// testing.
|
/// mostly for testing.
|
||||||
fn from(value: (u16, u16)) -> Self {
|
fn from(value: (u16, u16)) -> Self {
|
||||||
Self {
|
Self {
|
||||||
block: value.0,
|
block: value.0,
|
||||||
|
@ -41,69 +41,67 @@ impl From<(u16, u16)> for LimitIndices {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Contains a mapping of hosts to limits as well as a trust value which is
|
/// Contains a mapping of hosts to moderation actions as well as a trust value,
|
||||||
/// used to weight limits when building a merged list
|
/// which is used to weight moderation actions when building a mod map
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
pub struct LimitList {
|
pub struct ModSource {
|
||||||
pub limits: HashMap<String, Limit>,
|
pub actions: HashMap<String, ModAction>,
|
||||||
pub trust: u16,
|
pub trust: u16,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for LimitList {
|
impl Default for ModSource {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
limits: HashMap::new(),
|
actions: HashMap::new(),
|
||||||
trust: 100,
|
trust: 100,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<HashMap<String, Limit>> for LimitList {
|
impl From<HashMap<String, ModAction>> for ModSource {
|
||||||
fn from(map: HashMap<String, Limit>) -> Self {
|
fn from(map: HashMap<String, ModAction>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
limits: map,
|
actions: map,
|
||||||
trust: 100,
|
trust: 100,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LimitList {
|
impl ModSource {
|
||||||
fn add_host(&mut self, host: &str, limit: Limit) -> &mut Self {
|
fn add_action(&mut self, host: &str, action: ModAction) -> &mut Self {
|
||||||
self.limits.insert(host.to_string(), limit);
|
self.actions.insert(host.to_string(), action);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build(map: HashMap<String, Limit>, trust: u16) -> Self {
|
pub fn build(map: HashMap<String, ModAction>, trust: u16) -> Self {
|
||||||
let mut src = Self::from(map);
|
let mut src = Self::from(map);
|
||||||
src.trust = trust;
|
src.trust = trust;
|
||||||
src
|
src
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn import_file(&mut self, path: &str, limit: Limit) -> &mut Self {
|
pub fn import_file(&mut self, path: &str, action: ModAction) -> &mut Self {
|
||||||
let contents = fs::read_to_string(path).unwrap();
|
let contents = fs::read_to_string(path).unwrap();
|
||||||
for host in contents.lines().filter(|line| !line.is_empty()) {
|
for host in contents.lines().filter(|line| !line.is_empty()) {
|
||||||
self.add_host(host, limit);
|
self.add_action(host, action);
|
||||||
}
|
}
|
||||||
|
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A map of hosts (as strings) to their limit weights
|
/// A map of hosts (as strings) to weighted mod actions
|
||||||
#[derive(Debug, Default, Eq, PartialEq)]
|
#[derive(Debug, Default, Eq, PartialEq)]
|
||||||
pub struct MergedLimitList {
|
pub struct ModMap(pub HashMap<String, ModActionTrust>);
|
||||||
pub map: HashMap<String, LimitIndices>,
|
|
||||||
pub max: u16,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MergedLimitList {
|
impl ModMap {
|
||||||
pub fn add_limit_list(&mut self, src: LimitList) -> &mut Self {
|
pub fn add_source(&mut self, src: ModSource) -> &mut Self {
|
||||||
for (host, limit) in src.limits.into_iter() {
|
let items = src.actions.into_iter();
|
||||||
let entry = self.map.entry(host).or_default();
|
|
||||||
entry.add_limit(limit, src.trust);
|
for (host, action) in items {
|
||||||
|
let entry = self.0.entry(host).or_default();
|
||||||
|
entry.add_action(action, src.trust);
|
||||||
}
|
}
|
||||||
|
|
||||||
self.max += src.trust;
|
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,18 +109,18 @@ impl MergedLimitList {
|
||||||
self,
|
self,
|
||||||
block_path: &str,
|
block_path: &str,
|
||||||
mute_path: &str,
|
mute_path: &str,
|
||||||
indices: (u16, u16),
|
heat: (u16, u16),
|
||||||
) -> std::io::Result<()> {
|
) -> std::io::Result<()> {
|
||||||
if self.map.is_empty() {
|
if self.0.is_empty() {
|
||||||
error!("Nothing to export!");
|
error!("Nothing to export!");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let (block_thresh, mute_thresh) = indices;
|
let (block_thresh, mute_thresh) = heat;
|
||||||
let mut block_output: String = String::default();
|
let mut block_output: String = String::default();
|
||||||
let mut mute_output: String = String::default();
|
let mut mute_output: String = String::default();
|
||||||
|
|
||||||
for item in self.map.into_iter() {
|
for item in self.0.into_iter() {
|
||||||
let (block_trust, mute_trust) = (item.1.block, item.1.silence);
|
let (block_trust, mute_trust) = (item.1.block, item.1.silence);
|
||||||
|
|
||||||
if block_trust >= block_thresh {
|
if block_trust >= block_thresh {
|
||||||
|
|
|
@ -5,13 +5,13 @@ use std::{collections::HashMap, fs};
|
||||||
use crate::manip::*;
|
use crate::manip::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limit_add() {
|
fn modaction_add() {
|
||||||
let mut at = LimitIndices::default();
|
let mut at = ModActionTrust::default();
|
||||||
|
|
||||||
at.add_limit(Limit::Block, 123)
|
at.add_action(ModAction::Block, 123)
|
||||||
.add_limit(Limit::Silence, 456);
|
.add_action(ModAction::Silence, 456);
|
||||||
|
|
||||||
let test_at = LimitIndices {
|
let test_at = ModActionTrust {
|
||||||
block: 123,
|
block: 123,
|
||||||
silence: 456,
|
silence: 456,
|
||||||
};
|
};
|
||||||
|
@ -20,14 +20,14 @@ fn limit_add() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limit_combine() {
|
fn modaction_combine() {
|
||||||
let mut at = LimitIndices::default();
|
let mut at = ModActionTrust::default();
|
||||||
|
|
||||||
at.add_limit(Limit::Block, 123)
|
at.add_action(ModAction::Block, 123)
|
||||||
.add_limit(Limit::Block, 333)
|
.add_action(ModAction::Block, 333)
|
||||||
.add_limit(Limit::Silence, 123);
|
.add_action(ModAction::Silence, 123);
|
||||||
|
|
||||||
let test_at = LimitIndices {
|
let test_at = ModActionTrust {
|
||||||
block: 456,
|
block: 456,
|
||||||
silence: 123,
|
silence: 123,
|
||||||
};
|
};
|
||||||
|
@ -36,19 +36,19 @@ fn limit_combine() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limitlist_from_map() {
|
fn modsource_from_map() {
|
||||||
let src1 = LimitList::from(HashMap::from([
|
let src1 = ModSource::from(HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
src1.limits,
|
src1.actions,
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
])
|
])
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -56,133 +56,139 @@ fn limitlist_from_map() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limitlist_from_map_and_trust() {
|
fn modsource_from_map_and_trust() {
|
||||||
let src2 = LimitList::build(
|
let src2 = ModSource::build(
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
]),
|
]),
|
||||||
123,
|
123,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
src2.limits,
|
src2.actions,
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
])
|
])
|
||||||
);
|
);
|
||||||
assert_eq!(src2.trust, 123);
|
assert_eq!(src2.trust, 123);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limitlist_from_file() {
|
fn modsource_from_file() {
|
||||||
let mut src = LimitList::default();
|
let mut src = ModSource::default();
|
||||||
src.import_file("test/example_blocklist.txt", Limit::Block)
|
src.import_file("test/example_blocklist.txt", ModAction::Block)
|
||||||
.import_file("test/example_mutelist.txt", Limit::Silence);
|
.import_file("test/example_mutelist.txt", ModAction::Silence);
|
||||||
|
|
||||||
let test_src = LimitList::from(HashMap::from([
|
let test_src = ModSource::from(HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Block),
|
(String::from("example.org"), ModAction::Block),
|
||||||
(String::from("example.net"), Limit::Silence),
|
(String::from("example.net"), ModAction::Silence),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
assert_eq!(test_src, src);
|
assert_eq!(test_src, src);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mergedlist_from_limitlist() {
|
fn modmap_from_modsource() {
|
||||||
let mut ml = MergedLimitList::default();
|
let mut ml = ModMap::default();
|
||||||
|
|
||||||
let src1 = LimitList::from(HashMap::from([
|
let src1 = ModSource::from(HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
let mut src2 = LimitList::default();
|
let mut src2 = ModSource::default();
|
||||||
src2.import_file("test/example_blocklist.txt", Limit::Block)
|
src2.import_file("test/example_blocklist.txt", ModAction::Block)
|
||||||
.import_file("test/example_mutelist.txt", Limit::Silence);
|
.import_file("test/example_mutelist.txt", ModAction::Silence);
|
||||||
|
|
||||||
ml.add_limit_list(src1).add_limit_list(src2);
|
ml.add_source(src1).add_source(src2);
|
||||||
|
|
||||||
let test_ml = MergedLimitList {
|
let test_ml = ModMap(HashMap::from([
|
||||||
map: HashMap::from([
|
(String::from("example.com"), ModActionTrust::from((200, 0))),
|
||||||
(String::from("example.com"), LimitIndices::from((200, 0))),
|
(
|
||||||
(String::from("example.org"), LimitIndices::from((100, 100))),
|
String::from("example.org"),
|
||||||
(String::from("example.net"), LimitIndices::from((100, 100))),
|
ModActionTrust::from((100, 100)),
|
||||||
]),
|
),
|
||||||
max: 200,
|
(
|
||||||
};
|
String::from("example.net"),
|
||||||
|
ModActionTrust::from((100, 100)),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
|
||||||
assert_eq!(ml, test_ml);
|
assert_eq!(ml, test_ml);
|
||||||
|
|
||||||
let src3 = LimitList::build(
|
let src3 = ModSource::build(
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
]),
|
]),
|
||||||
200,
|
200,
|
||||||
);
|
);
|
||||||
|
|
||||||
let src4 = LimitList::build(
|
let src4 = ModSource::build(
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.net"), Limit::Silence),
|
(String::from("example.net"), ModAction::Silence),
|
||||||
]),
|
]),
|
||||||
50,
|
50,
|
||||||
);
|
);
|
||||||
|
|
||||||
ml.add_limit_list(src3).add_limit_list(src4);
|
ml.add_source(src3).add_source(src4);
|
||||||
|
|
||||||
let test_ml = MergedLimitList {
|
let test_ml = ModMap(HashMap::from([
|
||||||
map: HashMap::from([
|
(String::from("example.com"), ModActionTrust::from((450, 0))),
|
||||||
(String::from("example.com"), LimitIndices::from((450, 0))),
|
(
|
||||||
(String::from("example.org"), LimitIndices::from((100, 300))),
|
String::from("example.org"),
|
||||||
(String::from("example.net"), LimitIndices::from((100, 150))),
|
ModActionTrust::from((100, 300)),
|
||||||
]),
|
),
|
||||||
max: 450,
|
(
|
||||||
};
|
String::from("example.net"),
|
||||||
|
ModActionTrust::from((100, 150)),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
|
||||||
assert_eq!(ml, test_ml);
|
assert_eq!(ml, test_ml);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mergedlist_export_txt() {
|
fn modmap_export_txt() {
|
||||||
let mut ml = MergedLimitList::default();
|
let mut ml = ModMap::default();
|
||||||
|
|
||||||
let src1 = LimitList::from(HashMap::from([
|
let src1 = ModSource::from(HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
(String::from("example.net"), Limit::Block),
|
(String::from("example.net"), ModAction::Block),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
let mut src2 = LimitList::default();
|
let mut src2 = ModSource::default();
|
||||||
src2.import_file("test/example_blocklist.txt", Limit::Block)
|
src2.import_file("test/example_blocklist.txt", ModAction::Block)
|
||||||
.import_file("test/example_mutelist.txt", Limit::Silence);
|
.import_file("test/example_mutelist.txt", ModAction::Silence);
|
||||||
|
|
||||||
let src3 = LimitList::build(
|
let src3 = ModSource::build(
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.org"), Limit::Silence),
|
(String::from("example.org"), ModAction::Silence),
|
||||||
]),
|
]),
|
||||||
200,
|
200,
|
||||||
);
|
);
|
||||||
|
|
||||||
let src4 = LimitList::build(
|
let src4 = ModSource::build(
|
||||||
HashMap::from([
|
HashMap::from([
|
||||||
(String::from("example.com"), Limit::Block),
|
(String::from("example.com"), ModAction::Block),
|
||||||
(String::from("example.net"), Limit::Silence),
|
(String::from("example.net"), ModAction::Silence),
|
||||||
]),
|
]),
|
||||||
50,
|
50,
|
||||||
);
|
);
|
||||||
|
|
||||||
ml.add_limit_list(src1)
|
ml.add_source(src1)
|
||||||
.add_limit_list(src2)
|
.add_source(src2)
|
||||||
.add_limit_list(src3)
|
.add_source(src3)
|
||||||
.add_limit_list(src4);
|
.add_source(src4);
|
||||||
|
|
||||||
let _ = ml.export_file("test/test_blocks.txt", "test/test_mutes.txt", (200, 150));
|
let _ = ml.export_file("test/test_blocks.txt", "test/test_mutes.txt", (200, 150));
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue