Compare commits

...

2 commits

Author SHA1 Message Date
gil 1dd0083b5d Read from text files, reorganize tests 2024-05-30 01:59:40 -05:00
gil 88a0b80e45 Add README, rewrite key structs/methods 2024-05-30 00:26:14 -05:00
9 changed files with 269 additions and 144 deletions

12
README.md Normal file
View file

@ -0,0 +1,12 @@
# fediloom 🪢
Goal - a commandline-based tool for creating blocklists for ActivityPub software ("the Fediverse")
## Planned features
- [ ] Accept and parse commandline arguments
- [ ] Add ability to specify reasons
- [ ] Use URLs as sources
- [ ] Directly request lists from API endpoints
- [ ] Create tiered or thresholded lists
- [ ] Support CSV and JSON1

5
example_blocklist1.txt Normal file
View file

@ -0,0 +1,5 @@
example.com
example.org

3
example_mutelist1.txt Normal file
View file

@ -0,0 +1,3 @@
example.net

53
src/list.rs Normal file
View file

@ -0,0 +1,53 @@
use std::collections::HashMap;
mod source;
mod tests;
pub use source::*;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Action {
Block,
Silence,
}
#[derive(Debug, Default, Eq, PartialEq)]
pub struct ActionTrust {
pub block: u16,
pub silence: u16,
}
impl ActionTrust {
pub fn add_action(&mut self, action: Action, trust: u16) -> &mut Self {
match action {
Action::Block => self.block += trust,
Action::Silence => self.silence += trust,
}
self
}
}
impl From<(u16, u16)> for ActionTrust {
fn from(value: (u16, u16)) -> Self {
Self {
block: value.0,
silence: value.1,
}
}
}
#[derive(Debug, Default, Eq, PartialEq)]
pub struct ModerationList(pub HashMap<String, ActionTrust>);
impl ModerationList {
pub fn add_source(&mut self, src: Source) -> &mut Self {
let items = src.actions.into_iter();
for (host, action) in items {
let entry = self.0.entry(host).or_default();
entry.add_action(action, src.trust);
}
self
}
}

View file

@ -1,69 +0,0 @@
use std::collections::HashMap;
pub struct SourceList {
pub blocks: Vec<String>,
pub confidence: u8,
}
impl SourceList {
pub fn new() -> Self {
SourceList {
blocks: Vec::new(),
confidence: 100,
}
}
}
impl From<Vec<String>> for SourceList {
fn from(blocked: Vec<String>) -> Self {
SourceList {
blocks: blocked,
confidence: 100,
}
}
}
pub struct TrustMaps {
pub blocks: HashMap<String, usize>,
}
impl TrustMaps {
pub fn new() -> TrustMaps {
TrustMaps {
blocks: HashMap::new(),
}
}
pub fn add_source(&mut self, src: &SourceList) -> &mut TrustMaps {
for host in src.blocks.iter() {
let entry = self.blocks.entry(host.to_string()).or_insert(0);
*entry += src.confidence as usize;
}
self
}
pub fn remove_source(&mut self, src: &SourceList) -> &mut TrustMaps {
// If blocks already empty, nothing to remove so just return
if self.blocks.is_empty() {
return self;
}
// Iterate over host strings in source list
for host in src.blocks.iter() {
// If the host isn't in the output list, then skip
if !self.blocks.contains_key(host) {
continue;
}
// Host is valid entry, subtract confidence value and saturate at 0
self.blocks.entry(host.to_string()).and_modify(|v| {
*v = v.saturating_sub(src.confidence as usize);
});
}
// Finally, if any value became zero, remove the pair from the map
self.blocks.retain(|_, v| *v != 0);
self
}
}

49
src/list/source.rs Normal file
View file

@ -0,0 +1,49 @@
use std::{collections::HashMap, fs};
use super::*;
#[derive(Debug, PartialEq, Eq)]
pub struct Source {
pub actions: HashMap<String, Action>,
pub trust: u16,
}
impl Default for Source {
fn default() -> Self {
Self {
actions: HashMap::new(),
trust: 100,
}
}
}
impl From<HashMap<String, Action>> for Source {
fn from(map: HashMap<String, Action>) -> Self {
Self {
actions: map,
trust: 100,
}
}
}
impl Source {
pub fn build(map: HashMap<String, Action>, trust: u16) -> Self {
let mut src = Self::from(map);
src.trust = trust;
src
}
pub fn add_from_file(&mut self, path: &str, action: Action) -> &mut Self {
let contents = fs::read_to_string(path).unwrap();
for host in contents.lines().filter(|line| !line.is_empty()) {
self.add_action(host, action);
}
self
}
fn add_action(&mut self, host: &str, action: Action) -> &mut Self {
self.actions.insert(host.to_string(), action);
self
}
}

144
src/list/tests.rs Normal file
View file

@ -0,0 +1,144 @@
#![cfg(test)]
use std::collections::HashMap;
use super::*;
#[test]
fn add_action() {
let mut at = ActionTrust::default();
at.add_action(Action::Block, 123)
.add_action(Action::Silence, 456);
let test_at = ActionTrust {
block: 123,
silence: 456,
};
assert_eq!(at, test_at);
}
#[test]
fn add_action_overlap() {
let mut at = ActionTrust::default();
at.add_action(Action::Block, 123)
.add_action(Action::Block, 333)
.add_action(Action::Silence, 123);
let test_at = ActionTrust {
block: 456,
silence: 123,
};
assert_eq!(at, test_at);
}
#[test]
fn source_new_from_hashmap() {
let src1 = Source::from(HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
(String::from("example.net"), Action::Block),
]));
assert_eq!(
src1.actions,
HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
(String::from("example.net"), Action::Block),
])
);
assert_eq!(src1.trust, 100);
}
#[test]
fn source_build_from_hashmap() {
let src2 = Source::build(
HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
(String::from("example.net"), Action::Block),
]),
123,
);
assert_eq!(
src2.actions,
HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
(String::from("example.net"), Action::Block),
])
);
assert_eq!(src2.trust, 123);
}
#[test]
fn source_add_from_file() {
let mut src = Source::default();
src.add_from_file("example_blocklist1.txt", Action::Block)
.add_from_file("example_mutelist1.txt", Action::Silence);
let test_src = Source::from(HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Block),
(String::from("example.net"), Action::Silence),
]));
assert_eq!(test_src, src);
}
#[test]
fn build_moderation_list() {
let mut ml = ModerationList::default();
let src1 = Source::from(HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
(String::from("example.net"), Action::Block),
]));
let mut src2 = Source::default();
src2.add_from_file("example_blocklist1.txt", Action::Block)
.add_from_file("example_mutelist1.txt", Action::Silence);
ml.add_source(src1).add_source(src2);
let test_ml = ModerationList(HashMap::from([
(String::from("example.com"), ActionTrust::from((200, 0))),
(String::from("example.org"), ActionTrust::from((100, 100))),
(String::from("example.net"), ActionTrust::from((100, 100))),
]));
assert_eq!(ml, test_ml);
let src3 = Source::build(
HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.org"), Action::Silence),
]),
200,
);
let src4 = Source::build(
HashMap::from([
(String::from("example.com"), Action::Block),
(String::from("example.net"), Action::Silence),
]),
50,
);
ml.add_source(src3).add_source(src4);
let test_ml = ModerationList(HashMap::from([
(String::from("example.com"), ActionTrust::from((450, 0))),
(String::from("example.org"), ActionTrust::from((100, 300))),
(String::from("example.net"), ActionTrust::from((100, 150))),
]));
assert_eq!(ml, test_ml);
}

View file

@ -1,20 +1,10 @@
// src/main.rs
mod list;
mod tests;
// things we need to do:
// - import 2 or more lists
// - assign "trust" percent to each list
// - combine lists together and weight each element by trust %s
// - export 1 or more lists filtered by heat rating
// - tiered lists, e.g.:
// - tier 0: instances uniformly blocked
// - tier 1: instances mixed blocked and silenced
// - tier 2: instances silenced
// - tier 3: instances mixed silenced and no action
// - tier 4: instances no action (i.e., below all thresholds)
fn main() {
println!("Hello, world!");
// TODO argument parsing
// TODO logging
// TODO config file
}

View file

@ -1,62 +0,0 @@
#![cfg(test)]
use super::*;
use list::*;
use std::collections::HashMap;
#[test]
fn add_lists() {
let hosts1 = vec![
String::from("example.com"),
String::from("example.org"),
String::from("example.net"),
];
let hosts2 = vec![String::from("example.com"), String::from("example.org")];
let hosts3 = vec![String::from("example.com")];
let src1 = SourceList::from(hosts1);
let src2 = SourceList::from(hosts2);
let src3 = SourceList::from(hosts3);
let mut output = TrustMaps::new();
output.add_source(&src1).add_source(&src2).add_source(&src3);
let test_map = HashMap::from([
(String::from("example.com"), 300),
(String::from("example.org"), 200),
(String::from("example.net"), 100),
]);
assert_eq!(test_map, output.blocks);
}
#[test]
fn remove_lists() {
let hosts1 = vec![
String::from("example.com"),
String::from("example.org"),
String::from("example.net"),
];
let hosts2 = vec![String::from("example.com"), String::from("example.org")];
let hosts3 = vec![String::from("example.com")];
let src1 = SourceList::from(hosts1);
let src2 = SourceList::from(hosts2);
let src3 = SourceList::from(hosts3);
let mut output = TrustMaps::new();
output
.add_source(&src1)
.add_source(&src2)
.add_source(&src3)
.remove_source(&src3)
.remove_source(&src3)
.remove_source(&src3)
.remove_source(&src3);
let test_map = HashMap::from([
(String::from("example.org"), 200),
(String::from("example.net"), 100),
]);
assert_eq!(test_map, output.blocks);
}